You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2020/07/01 23:08:14 UTC
[hbase] branch branch-2.3 updated (a2dbff0 -> 58a9ff5)
This is an automated email from the ASF dual-hosted git repository.
stack pushed a change to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from a2dbff0 HBASE-24656 [Flakey Tests] branch-2 TestMasterNoCluster.testStopDuringStart (#1998)
new bf8749b HBASE-24574 Procedure V2 - Distributed WAL Splitting => LOGGING (#1912)
new 58a9ff5 HBASE-24574 Procedure V2 - Distributed WAL Splitting => LOGGING (#1912) Addendum; minor log edits
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/hbase/master/SplitLogManager.java | 31 +++++++++++-------
.../hadoop/hbase/master/SplitWALManager.java | 17 ++++++----
.../hbase/master/assignment/AssignmentManager.java | 8 +++--
.../assignment/RegionRemoteProcedureBase.java | 20 ++++++++++--
.../assignment/RegionTransitionProcedure.java | 15 ++++-----
.../master/procedure/ServerCrashProcedure.java | 8 +++--
.../hbase/master/procedure/SplitWALProcedure.java | 37 ++++++++++++++++++----
.../master/procedure/SplitWALRemoteProcedure.java | 31 +++++++++++-------
.../hadoop/hbase/regionserver/HRegionServer.java | 4 +--
.../RemoteProcedureResultReporter.java | 6 ++--
.../hadoop/hbase/regionserver/SplitLogWorker.java | 18 ++++-------
.../hbase/regionserver/SplitWALCallable.java | 2 +-
.../regionserver/handler/WALSplitterHandler.java | 1 +
.../hbase/regionserver/wal/AbstractFSWAL.java | 3 +-
14 files changed, 129 insertions(+), 72 deletions(-)
[hbase] 02/02: HBASE-24574 Procedure V2 - Distributed WAL Splitting
=> LOGGING (#1912) Addendum; minor log edits
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 58a9ff59b2801c9003661eaf3977e9825072aa31
Author: stack <st...@apache.org>
AuthorDate: Thu Jun 18 08:37:19 2020 -0700
HBASE-24574 Procedure V2 - Distributed WAL Splitting => LOGGING (#1912)
Addendum; minor log edits
---
.../hbase/master/procedure/SplitWALProcedure.java | 7 +++++--
.../regionserver/RemoteProcedureResultReporter.java | 6 +++---
.../hadoop/hbase/regionserver/SplitLogWorker.java | 18 +++++++-----------
.../hbase/regionserver/handler/WALSplitterHandler.java | 1 +
4 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java
index 81d525d..4ae408f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java
@@ -185,8 +185,11 @@ public class SplitWALProcedure
@Override
protected void afterReplay(MasterProcedureEnv env){
- if(worker != null){
- env.getMasterServices().getSplitWALManager().addUsedSplitWALWorker(worker);
+ if (worker != null) {
+ if (env != null && env.getMasterServices() != null &&
+ env.getMasterServices().getSplitWALManager() != null) {
+ env.getMasterServices().getSplitWALManager().addUsedSplitWALWorker(worker);
+ }
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
index efb044a..981f090 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
@@ -57,11 +57,11 @@ class RemoteProcedureResultReporter extends Thread {
public void complete(long procId, Throwable error) {
RemoteProcedureResult.Builder builder = RemoteProcedureResult.newBuilder().setProcId(procId);
if (error != null) {
- LOG.debug("Failed to complete execution of proc pid={}", procId, error);
+ LOG.debug("Failed to complete execution of pid={}", procId, error);
builder.setStatus(RemoteProcedureResult.Status.ERROR).setError(
ForeignExceptionUtil.toProtoForeignException(server.getServerName().toString(), error));
} else {
- LOG.debug("Successfully complete execution of proc pid={}", procId);
+ LOG.debug("Successfully complete execution of pid={}", procId);
builder.setStatus(RemoteProcedureResult.Status.SUCCESS);
}
results.add(builder.build());
@@ -102,7 +102,7 @@ class RemoteProcedureResultReporter extends Thread {
} else {
pauseTime = INIT_PAUSE_TIME_MS; // Reset.
}
- LOG.info("Failed report procedure " + TextFormat.shortDebugString(request) + "; retry (#" +
+ LOG.info("Failed procedure report " + TextFormat.shortDebugString(request) + "; retry (#" +
tries + ")" + (pause ? " after " + pauseTime + "ms delay (Master is coming online...)."
: " immediately."),
e);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
index 6a8e288..44c8f5c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
@@ -1,5 +1,4 @@
-/**
- *
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -39,7 +38,6 @@ import org.apache.hadoop.hbase.wal.WALSplitter;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
@@ -95,7 +93,7 @@ public class SplitLogWorker implements Runnable {
walDir = CommonFSUtils.getWALRootDir(conf);
fs = walDir.getFileSystem(conf);
} catch (IOException e) {
- LOG.warn("could not find root dir or fs", e);
+ LOG.warn("Resigning, could not find root dir or fs", e);
return Status.RESIGNED;
}
// TODO have to correctly figure out when log splitting has been
@@ -110,26 +108,24 @@ public class SplitLogWorker implements Runnable {
return Status.PREEMPTED;
}
} catch (InterruptedIOException iioe) {
- LOG.warn("log splitting of " + filename + " interrupted, resigning", iioe);
+ LOG.warn("Resigning, interrupted splitting WAL {}", filename, iioe);
return Status.RESIGNED;
} catch (IOException e) {
if (e instanceof FileNotFoundException) {
// A wal file may not exist anymore. Nothing can be recovered so move on
- LOG.warn("WAL {} does not exist anymore", filename, e);
+ LOG.warn("Done, WAL {} does not exist anymore", filename, e);
return Status.DONE;
}
Throwable cause = e.getCause();
if (e instanceof RetriesExhaustedException && (cause instanceof NotServingRegionException
|| cause instanceof ConnectException || cause instanceof SocketTimeoutException)) {
- LOG.warn("log replaying of " + filename + " can't connect to the target regionserver, "
- + "resigning",
- e);
+ LOG.warn("Resigning, can't connect to target regionserver splitting WAL {}", filename, e);
return Status.RESIGNED;
} else if (cause instanceof InterruptedException) {
- LOG.warn("log splitting of " + filename + " interrupted, resigning", e);
+ LOG.warn("Resigning, interrupted splitting WAL {}", filename, e);
return Status.RESIGNED;
}
- LOG.warn("log splitting of " + filename + " failed, returning error", e);
+ LOG.warn("Error splitting WAL {}", filename, e);
return Status.ERR;
}
LOG.debug("Done splitting WAL {}", filename);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/WALSplitterHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/WALSplitterHandler.java
index 49ab574..d6009e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/WALSplitterHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/WALSplitterHandler.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.util.CancelableProgressable;
/**
* Handles log splitting a wal
+ * Used by the zk-based distributed log splitting. Created by ZKSplitLogWorkerCoordination.
*/
@InterfaceAudience.Private
public class WALSplitterHandler extends EventHandler {
[hbase] 01/02: HBASE-24574 Procedure V2 - Distributed WAL Splitting
=> LOGGING (#1912)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit bf8749b3765bde3494af01f1171a1306d70bd491
Author: Michael Stack <sa...@users.noreply.github.com>
AuthorDate: Thu Jun 18 08:32:48 2020 -0700
HBASE-24574 Procedure V2 - Distributed WAL Splitting => LOGGING (#1912)
hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
Don't register as a chore on construction if no coordination state
manager instance (there is no instance when procv2 WAL splitter).
hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java
hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
Edit logs.
hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
Add proc name rather than rely on default behavior. Add detail to the
toString.
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Factoring
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
Print the maxLogs... we don't do it any where.
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
Utility method to strip prefix from wal path.
Signed-off-by: Guanghao Zhang <zg...@apache.org>
Signed-off-by: tianjingyun <ti...@apache.org>
---
.../hadoop/hbase/master/SplitLogManager.java | 31 +++++++++++++---------
.../hadoop/hbase/master/SplitWALManager.java | 17 +++++++-----
.../hbase/master/assignment/AssignmentManager.java | 8 +++---
.../assignment/RegionRemoteProcedureBase.java | 20 +++++++++++---
.../assignment/RegionTransitionProcedure.java | 15 +++++------
.../master/procedure/ServerCrashProcedure.java | 8 +++---
.../hbase/master/procedure/SplitWALProcedure.java | 30 +++++++++++++++++----
.../master/procedure/SplitWALRemoteProcedure.java | 31 ++++++++++++++--------
.../hadoop/hbase/regionserver/HRegionServer.java | 4 +--
.../hbase/regionserver/SplitWALCallable.java | 2 +-
.../hbase/regionserver/wal/AbstractFSWAL.java | 3 ++-
11 files changed, 113 insertions(+), 56 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 5ac75d9..7ed0d9a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,14 +16,12 @@
* limitations under the License.
*/
package org.apache.hadoop.hbase.master;
-
import static org.apache.hadoop.hbase.master.SplitLogManager.ResubmitDirective.CHECK;
import static org.apache.hadoop.hbase.master.SplitLogManager.ResubmitDirective.FORCE;
import static org.apache.hadoop.hbase.master.SplitLogManager.TerminationStatus.DELETED;
import static org.apache.hadoop.hbase.master.SplitLogManager.TerminationStatus.FAILURE;
import static org.apache.hadoop.hbase.master.SplitLogManager.TerminationStatus.IN_PROGRESS;
import static org.apache.hadoop.hbase.master.SplitLogManager.TerminationStatus.SUCCESS;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -57,7 +55,6 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
@@ -88,6 +85,7 @@ import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesti
* completed (either with success or with error) it will be not be submitted
* again. If a task is resubmitted then there is a risk that old "delete task"
* can delete the re-submission.
+ * @see SplitWALManager for an alternate implementation based on Procedures.
*/
@InterfaceAudience.Private
public class SplitLogManager {
@@ -121,21 +119,28 @@ public class SplitLogManager {
throws IOException {
this.server = master;
this.conf = conf;
- this.choreService =
- new ChoreService(master.getServerName().toShortString() + ".splitLogManager.");
+ // If no CoordinatedStateManager, skip registering as a chore service (The
+ // CoordinatedStateManager is non-null if we are running the ZK-based distributed WAL
+ // splitting. It is null if we are configured to use procedure-based distributed WAL
+ // splitting.
if (server.getCoordinatedStateManager() != null) {
+ this.choreService =
+ new ChoreService(master.getServerName().toShortString() + ".splitLogManager.");
SplitLogManagerCoordination coordination = getSplitLogManagerCoordination();
Set<String> failedDeletions = Collections.synchronizedSet(new HashSet<String>());
SplitLogManagerDetails details = new SplitLogManagerDetails(tasks, master, failedDeletions);
coordination.setDetails(details);
coordination.init();
- }
- this.unassignedTimeout =
+ this.unassignedTimeout =
conf.getInt("hbase.splitlog.manager.unassigned.timeout", DEFAULT_UNASSIGNED_TIMEOUT);
- this.timeoutMonitor =
+ this.timeoutMonitor =
new TimeoutMonitor(conf.getInt("hbase.splitlog.manager.timeoutmonitor.period", 1000),
- master);
- choreService.scheduleChore(timeoutMonitor);
+ master);
+ this.choreService.scheduleChore(timeoutMonitor);
+ } else {
+ this.choreService = null;
+ this.timeoutMonitor = null;
+ }
}
private SplitLogManagerCoordination getSplitLogManagerCoordination() {
@@ -560,7 +565,9 @@ public class SplitLogManager {
@Override
protected void chore() {
- if (server.getCoordinatedStateManager() == null) return;
+ if (server.getCoordinatedStateManager() == null) {
+ return;
+ }
int resubmitted = 0;
int unassigned = 0;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java
index fc50840..76407e0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitWALManager.java
@@ -67,6 +67,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
* {@link SplitLogManager}, {@link org.apache.hadoop.hbase.zookeeper.ZKSplitLog} and
* {@link org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination} can be removed
* after we switch to procedure-based WAL splitting.
+ * @see SplitLogManager for the original distributed split WAL manager.
*/
@InterfaceAudience.Private
public class SplitWALManager {
@@ -96,7 +97,7 @@ public class SplitWALManager {
// 2. create corresponding procedures
return createSplitWALProcedures(splittingFiles, crashedServer);
} catch (IOException e) {
- LOG.error("failed to create procedures for splitting logs of {}", crashedServer, e);
+ LOG.error("Failed to create procedures for splitting WALs of {}", crashedServer, e);
throw e;
}
}
@@ -106,7 +107,7 @@ public class SplitWALManager {
List<Path> logDirs = master.getMasterWalManager().getLogDirs(Collections.singleton(serverName));
FileStatus[] fileStatuses =
SplitLogManager.getFileList(this.conf, logDirs, splitMeta ? META_FILTER : NON_META_FILTER);
- LOG.info("size of WALs of {} is {}, isMeta: {}", serverName, fileStatuses.length, splitMeta);
+ LOG.info("{} WAL count={}, meta={}", serverName, fileStatuses.length, splitMeta);
return Lists.newArrayList(fileStatuses);
}
@@ -122,7 +123,9 @@ public class SplitWALManager {
public void deleteWALDir(ServerName serverName) throws IOException {
Path splitDir = getWALSplitDir(serverName);
- fs.delete(splitDir, false);
+ if (!fs.delete(splitDir, false)) {
+ LOG.warn("Failed delete {}", splitDir);
+ }
}
public boolean isSplitWALFinished(String walPath) throws IOException {
@@ -138,17 +141,17 @@ public class SplitWALManager {
}
/**
- * try to acquire an worker from online servers which is executring
+ * Acquire a split WAL worker
* @param procedure split WAL task
* @return an available region server which could execute this task
* @throws ProcedureSuspendedException if there is no available worker,
- * it will throw this exception to let the procedure wait
+ * it will throw this exception to WAIT the procedure.
*/
public ServerName acquireSplitWALWorker(Procedure<?> procedure)
throws ProcedureSuspendedException {
Optional<ServerName> worker = splitWorkerAssigner.acquire();
- LOG.debug("acquired a worker {} to split a WAL", worker);
if (worker.isPresent()) {
+ LOG.debug("Acquired split WAL worker={}", worker.get());
return worker.get();
}
splitWorkerAssigner.suspend(procedure);
@@ -162,7 +165,7 @@ public class SplitWALManager {
* @param scheduler scheduler which is to wake up the procedure event
*/
public void releaseSplitWALWorker(ServerName worker, MasterProcedureScheduler scheduler) {
- LOG.debug("release a worker {} to split a WAL", worker);
+ LOG.debug("Release split WAL worker={}", worker);
splitWorkerAssigner.release(worker);
splitWorkerAssigner.wake(scheduler);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 1df8678..04529f0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -1581,7 +1581,8 @@ public class AssignmentManager {
ProcedureExecutor<MasterProcedureEnv> procExec = this.master.getMasterProcedureExecutor();
carryingMeta = isCarryingMeta(serverName);
if (!force && serverNode != null && !serverNode.isInState(ServerState.ONLINE)) {
- LOG.info("Skip adding SCP for {} (meta={}) -- running?", serverNode, carryingMeta);
+ LOG.info("Skip adding ServerCrashProcedure for {} (meta={}) -- running?",
+ serverNode, carryingMeta);
return Procedure.NO_PROC_ID;
} else {
MasterProcedureEnv mpe = procExec.getEnvironment();
@@ -1602,8 +1603,9 @@ public class AssignmentManager {
pid = procExec.submitProcedure(
new ServerCrashProcedure(mpe, serverName, shouldSplitWal, carryingMeta));
}
- LOG.info("Scheduled SCP pid={} for {} (carryingMeta={}){}.", pid, serverName, carryingMeta,
- serverNode == null? "": " " + serverNode.toString() + ", oldState=" + oldState);
+ LOG.info("Scheduled ServerCrashProcedure pid={} for {} (carryingMeta={}){}.",
+ pid, serverName, carryingMeta,
+ serverNode == null? "": " " + serverNode.toString() + ", oldState=" + oldState);
}
} finally {
if (serverNode != null) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
index ee37746..1c90d81 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.master.assignment;
import java.io.IOException;
import java.util.Optional;
-
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
@@ -41,7 +40,6 @@ import org.apache.hadoop.hbase.util.RetryCounter;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseState;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionRemoteProcedureBaseStateData;
@@ -365,4 +363,20 @@ public abstract class RegionRemoteProcedureBase extends Procedure<MasterProcedur
protected void afterReplay(MasterProcedureEnv env) {
getParent(env).attachRemoteProc(this);
}
+
+ @Override public String getProcName() {
+ return getClass().getSimpleName() + " " + region.getEncodedName();
+ }
+
+ @Override protected void toStringClassDetails(StringBuilder builder) {
+ builder.append(getProcName());
+ if (targetServer != null) {
+ builder.append(", server=");
+ builder.append(this.targetServer);
+ }
+ if (this.retryCounter != null) {
+ builder.append(", retry=");
+ builder.append(this.retryCounter);
+ }
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index 59f2fbf..2e76771 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -32,9 +32,7 @@ import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperat
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureException;
import org.apache.yetus.audience.InterfaceAudience;
-
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionTransitionState;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
@@ -94,11 +92,12 @@ public abstract class RegionTransitionProcedure extends Procedure<MasterProcedur
@Override
public void toStringClassDetails(final StringBuilder sb) {
- sb.append(getClass().getSimpleName());
- sb.append(" table=");
- sb.append(getTableName());
- sb.append(", region=");
- sb.append(getRegionInfo() == null ? null : getRegionInfo().getEncodedName());
+ sb.append(getProcName());
+ }
+
+ @Override public String getProcName() {
+ RegionInfo r = getRegionInfo();
+ return getClass().getSimpleName() + " " + getTableName() + (r != null? r.getEncodedName(): "");
}
public RegionStateNode getRegionState(final MasterProcedureEnv env) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
index 3bc5854..a6ebbaa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
@@ -381,15 +381,17 @@ public class ServerCrashProcedure
@Override
public void toStringClassDetails(StringBuilder sb) {
- sb.append(getClass().getSimpleName());
- sb.append(" server=");
- sb.append(serverName);
+ sb.append(getProcName());
sb.append(", splitWal=");
sb.append(shouldSplitWal);
sb.append(", meta=");
sb.append(carryingMeta);
}
+ @Override public String getProcName() {
+ return getClass().getSimpleName() + " " + this.serverName;
+ }
+
@Override
protected void serializeStateData(ProcedureStateSerializer serializer)
throws IOException {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java
index fd741b5..81d525d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java
@@ -1,5 +1,4 @@
-/**
- *
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,9 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.hbase.master.procedure;
-
import java.io.IOException;
-
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.SplitWALManager;
@@ -33,7 +30,6 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
@@ -192,6 +188,30 @@ public class SplitWALProcedure
if(worker != null){
env.getMasterServices().getSplitWALManager().addUsedSplitWALWorker(worker);
}
+ }
+
+ @Override protected void toStringClassDetails(StringBuilder builder) {
+ builder.append(getProcName());
+ if (this.worker != null) {
+ builder.append(", worker=");
+ builder.append(this.worker);
+ }
+ if (this.retryCounter != null) {
+ builder.append(", retry=");
+ builder.append(this.retryCounter);
+ }
+ }
+
+ @Override public String getProcName() {
+ return getClass().getSimpleName() + " " + getWALNameFromStrPath(getWAL());
+ }
+ /**
+ * @return Return the WAL filename when given a Path-as-a-string; i.e. return the last path
+ * component only.
+ */
+ static String getWALNameFromStrPath(String path) {
+ int slashIndex = path.lastIndexOf('/');
+ return slashIndex != -1? path.substring(slashIndex + 1): path;
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
index c0bd92a..c829e51 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALRemoteProcedure.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.master.procedure;
import java.io.IOException;
import java.util.Optional;
-
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.ServerName;
@@ -31,14 +30,14 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
+
/**
* A remote procedure which is used to send split WAL request to region server.
- * it will return null if the task is succeed or return a DoNotRetryIOException
- * {@link SplitWALProcedure} will help handle the situation that encounter
- * DoNotRetryIOException. Otherwise it will retry until succeed.
+ * It will return null if the task succeeded or return a DoNotRetryIOException.
+ * {@link SplitWALProcedure} will help handle the situation that encounters
+ * DoNotRetryIOException. Otherwise it will retry until success.
*/
@InterfaceAudience.Private
public class SplitWALRemoteProcedure extends ServerRemoteProcedure
@@ -96,20 +95,18 @@ public class SplitWALRemoteProcedure extends ServerRemoteProcedure
@Override
protected void complete(MasterProcedureEnv env, Throwable error) {
if (error == null) {
- LOG.info("split WAL {} on {} succeeded", walPath, targetServer);
try {
env.getMasterServices().getSplitWALManager().deleteSplitWAL(walPath);
} catch (IOException e) {
- LOG.warn("remove WAL {} failed, ignore...", walPath, e);
+ LOG.warn("Failed split of {}; ignore...", walPath, e);
}
succ = true;
} else {
if (error instanceof DoNotRetryIOException) {
- LOG.warn("WAL split task of {} send to a wrong server {}, will retry on another server",
- walPath, targetServer, error);
+ LOG.warn("Sent {} to wrong server {}, try another", walPath, targetServer, error);
succ = true;
} else {
- LOG.warn("split WAL {} failed, retry...", walPath, error);
+ LOG.warn("Failed split of {}, retry...", walPath, error);
succ = false;
}
}
@@ -134,4 +131,16 @@ public class SplitWALRemoteProcedure extends ServerRemoteProcedure
public ServerOperationType getServerOperationType() {
return ServerOperationType.SPLIT_WAL_REMOTE;
}
+
+ @Override protected void toStringClassDetails(StringBuilder builder) {
+ builder.append(getProcName());
+ if (this.targetServer != null) {
+ builder.append(", worker=");
+ builder.append(this.targetServer);
+ }
+ }
+
+ @Override public String getProcName() {
+ return getClass().getSimpleName() + " " + SplitWALProcedure.getWALNameFromStrPath(getWAL());
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index a830277..5653879 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1964,9 +1964,9 @@ public class HRegionServer extends Thread implements
// Default is 2 mins. The default value for TTLCleaner is 5 mins so we set this to
// 2 mins so that compacted files can be archived before the TTLCleaner runs
int cleanerInterval =
- conf.getInt("hbase.hfile.compaction.discharger.interval", 2 * 60 * 1000);
+ conf.getInt("hbase.hfile.compaction.discharger.interval", 2 * 60 * 1000);
this.compactedFileDischarger =
- new CompactedHFilesDischarger(cleanerInterval, this, this);
+ new CompactedHFilesDischarger(cleanerInterval, this, this);
choreService.scheduleChore(compactedFileDischarger);
// Start executor services
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitWALCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitWALCallable.java
index 7790f13..03f2061 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitWALCallable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitWALCallable.java
@@ -61,7 +61,7 @@ public class SplitWALCallable implements RSProcedureCallable {
MasterProcedureProtos.SplitWALParameter.parseFrom(parameter);
this.walPath = param.getWalPath();
} catch (InvalidProtocolBufferException e) {
- LOG.error("parse proto buffer of split WAL request failed ", e);
+ LOG.error("Parse proto buffer of split WAL request failed ", e);
initError = e;
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index d0d2dcd..bf53352 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -443,7 +443,8 @@ public abstract class AbstractFSWAL<W extends WriterBase> implements WAL {
LOG.info("WAL configuration: blocksize=" + StringUtils.byteDesc(blocksize) + ", rollsize=" +
StringUtils.byteDesc(this.logrollsize) + ", prefix=" + this.walFilePrefix + ", suffix=" +
- walFileSuffix + ", logDir=" + this.walDir + ", archiveDir=" + this.walArchiveDir);
+ walFileSuffix + ", logDir=" + this.walDir + ", archiveDir=" + this.walArchiveDir +
+ ", maxLogs=" + this.maxLogs);
this.slowSyncNs = TimeUnit.MILLISECONDS.toNanos(conf.getInt(SLOW_SYNC_TIME_MS,
conf.getInt("hbase.regionserver.hlog.slowsync.ms", DEFAULT_SLOW_SYNC_TIME_MS)));
this.rollOnSyncNs = TimeUnit.MILLISECONDS.toNanos(conf.getInt(ROLL_ON_SYNC_TIME_MS,