You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2015/02/12 23:49:50 UTC
[9/9] hbase git commit: HBASE-9531 a command line (hbase shell)
interface to retreive the replication metrics and show replication lag
HBASE-9531 a command line (hbase shell) interface to retreive the replication metrics and show replication lag
Signed-off-by: Andrew Purtell <ap...@apache.org>
Conflicts:
hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c391dfbd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c391dfbd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c391dfbd
Branch: refs/heads/0.98
Commit: c391dfbd75de3f4d53d052b88048d833289901b6
Parents: 0ffb197
Author: Ashish Singhi <as...@huawei.com>
Authored: Thu Feb 12 14:00:19 2015 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Feb 12 14:28:07 2015 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/ServerLoad.java | 27 +-
.../hadoop/hbase/protobuf/ProtobufUtil.java | 24 +
.../hbase/replication/ReplicationLoadSink.java | 36 +
.../replication/ReplicationLoadSource.java | 53 +
.../MetricsReplicationSinkSource.java | 1 +
.../MetricsReplicationSourceSource.java | 1 +
.../MetricsReplicationGlobalSourceSource.java | 5 +
.../MetricsReplicationSinkSourceImpl.java | 5 +
.../MetricsReplicationSourceSourceImpl.java | 5 +
.../protobuf/generated/ClusterStatusProtos.java | 5535 +++++++++++++-----
.../src/main/protobuf/ClusterStatus.proto | 23 +
.../hbase/regionserver/HRegionServer.java | 17 +
.../hbase/regionserver/ReplicationService.java | 8 +-
.../replication/regionserver/MetricsSink.java | 17 +
.../replication/regionserver/MetricsSource.java | 34 +
.../replication/regionserver/Replication.java | 30 +
.../regionserver/ReplicationLoad.java | 151 +
.../regionserver/ReplicationSink.java | 8 +
.../regionserver/ReplicationSource.java | 8 +
.../replication/TestReplicationSmallTests.java | 49 +-
hbase-shell/src/main/ruby/hbase/admin.rb | 42 +-
.../src/main/ruby/shell/commands/status.rb | 9 +-
hbase-shell/src/test/ruby/hbase/admin_test.rb | 12 +
hbase-shell/src/test/ruby/test_helper.rb | 4 +
24 files changed, 4472 insertions(+), 1632 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c391dfbd/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
index 4c69e6d..bea9d5c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java
@@ -22,9 +22,12 @@ package org.apache.hadoop.hbase;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor;
+import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
+import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Strings;
@@ -53,7 +56,7 @@ public class ServerLoad {
private int totalStaticBloomSizeKB = 0;
private long totalCompactingKVs = 0;
private long currentCompactedKVs = 0;
-
+
public ServerLoad(ClusterStatusProtos.ServerLoad serverLoad) {
this.serverLoad = serverLoad;
for (ClusterStatusProtos.RegionLoad rl: serverLoad.getRegionLoadsList()) {
@@ -71,7 +74,7 @@ public class ServerLoad {
totalCompactingKVs += rl.getTotalCompactingKVs();
currentCompactedKVs += rl.getCurrentCompactedKVs();
}
-
+
}
// NOTE: Function name cannot start with "get" because then an OpenDataException is thrown because
@@ -179,6 +182,26 @@ public class ServerLoad {
}
/**
+ * Call directly from client such as hbase shell
+ * @return the list of ReplicationLoadSource
+ */
+ public List<ReplicationLoadSource> getReplicationLoadSourceList() {
+ return ProtobufUtil.toReplicationLoadSourceList(serverLoad.getReplLoadSourceList());
+ }
+
+ /**
+ * Call directly from client such as hbase shell
+ * @return ReplicationLoadSink
+ */
+ public ReplicationLoadSink getReplicationLoadSink() {
+ if (serverLoad.hasReplLoadSink()) {
+ return ProtobufUtil.toReplicationLoadSink(serverLoad.getReplLoadSink());
+ } else {
+ return null;
+ }
+ }
+
+ /**
* Originally, this method factored in the effect of requests going to the
* server as well. However, this does not interact very well with the current
* region rebalancing code, which only factors number of regions. For the
http://git-wip-us.apache.org/repos/asf/hbase/blob/c391dfbd/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 5f76bb1..e19c5b3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -95,6 +95,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.Col
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
+import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
@@ -110,6 +111,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
+import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
+import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.security.access.TablePermission;
import org.apache.hadoop.hbase.security.access.UserPermission;
@@ -2787,4 +2790,25 @@ public final class ProtobufUtil {
}
return result;
}
+
+ public static ReplicationLoadSink toReplicationLoadSink(
+ ClusterStatusProtos.ReplicationLoadSink cls) {
+ return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp());
+ }
+
+ public static ReplicationLoadSource toReplicationLoadSource(
+ ClusterStatusProtos.ReplicationLoadSource cls) {
+ return new ReplicationLoadSource(cls.getPeerID(), cls.getAgeOfLastShippedOp(),
+ cls.getSizeOfLogQueue(), cls.getTimeStampOfLastShippedOp(), cls.getReplicationLag());
+ }
+
+ public static List<ReplicationLoadSource> toReplicationLoadSourceList(
+ List<ClusterStatusProtos.ReplicationLoadSource> clsList) {
+ ArrayList<ReplicationLoadSource> rlsList = new ArrayList<ReplicationLoadSource>();
+ for (ClusterStatusProtos.ReplicationLoadSource cls : clsList) {
+ rlsList.add(toReplicationLoadSource(cls));
+ }
+ return rlsList;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c391dfbd/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java
new file mode 100644
index 0000000..63fe334
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * A HBase ReplicationLoad to present MetricsSink information
+ */
+@InterfaceAudience.Private
+public class ReplicationLoadSink {
+ private long ageOfLastAppliedOp;
+ private long timeStampsOfLastAppliedOp;
+
+ public ReplicationLoadSink(long age, long timeStamp) {
+ this.ageOfLastAppliedOp = age;
+ this.timeStampsOfLastAppliedOp = timeStamp;
+ }
+
+ public long getAgeOfLastAppliedOp() {
+ return this.ageOfLastAppliedOp;
+ }
+
+ public long getTimeStampsOfLastAppliedOp() {
+ return this.timeStampsOfLastAppliedOp;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c391dfbd/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java
new file mode 100644
index 0000000..bfd1599
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSource.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * A HBase ReplicationLoad to present MetricsSource information
+ */
+@InterfaceAudience.Private
+public class ReplicationLoadSource {
+ private String peerID;
+ private long ageOfLastShippedOp;
+ private int sizeOfLogQueue;
+ private long timeStampOfLastShippedOp;
+ private long replicationLag;
+
+ public ReplicationLoadSource(String id, long age, int size, long timeStamp, long lag) {
+ this.peerID = id;
+ this.ageOfLastShippedOp = age;
+ this.sizeOfLogQueue = size;
+ this.timeStampOfLastShippedOp = timeStamp;
+ this.replicationLag = lag;
+ }
+
+ public String getPeerID() {
+ return this.peerID;
+ }
+
+ public long getAgeOfLastShippedOp() {
+ return this.ageOfLastShippedOp;
+ }
+
+ public long getSizeOfLogQueue() {
+ return this.sizeOfLogQueue;
+ }
+
+ public long getTimeStampOfLastShippedOp() {
+ return this.timeStampOfLastShippedOp;
+ }
+
+ public long getReplicationLag() {
+ return this.replicationLag;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c391dfbd/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
index 50af94e..0aeb2af 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java
@@ -25,4 +25,5 @@ public interface MetricsReplicationSinkSource {
void setLastAppliedOpAge(long age);
void incrAppliedBatches(long batches);
void incrAppliedOps(long batchsize);
+ long getLastAppliedOpAge();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c391dfbd/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
index 66d265a..fecf191 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
@@ -43,4 +43,5 @@ public interface MetricsReplicationSourceSource {
void incrLogReadInBytes(long size);
void incrLogReadInEdits(long size);
void clear();
+ long getLastShippedAge();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c391dfbd/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index 5c9261e..08baf23 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -93,4 +93,9 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
@Override public void clear() {
}
+
+ @Override
+ public long getLastShippedAge() {
+ return ageOfLastShippedOpGauge.value();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c391dfbd/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
index 3025e3e..14212ba 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
@@ -44,4 +44,9 @@ public class MetricsReplicationSinkSourceImpl implements MetricsReplicationSinkS
@Override public void incrAppliedOps(long batchsize) {
opsCounter.incr(batchsize);
}
+
+ @Override
+ public long getLastAppliedOpAge() {
+ return ageGauge.value();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c391dfbd/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
index b4c631c..d5e52b8 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
@@ -126,4 +126,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
rms.removeMetric(logEditsFilteredKey);
}
+
+ @Override
+ public long getLastShippedAge() {
+ return ageOfLastShippedOpGauge.value();
+ }
}