You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2015/06/26 17:48:05 UTC
hbase git commit: HBASE-13863 Multi-wal feature breaks reported
number and size of HLogs (Abhilash)
Repository: hbase
Updated Branches:
refs/heads/branch-1 1650b228d -> 0426a5c70
HBASE-13863 Multi-wal feature breaks reported number and size of HLogs (Abhilash)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0426a5c7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0426a5c7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0426a5c7
Branch: refs/heads/branch-1
Commit: 0426a5c7094444b97855f95a2f9e51b5d111732c
Parents: 1650b22
Author: tedyu <yu...@gmail.com>
Authored: Fri Jun 26 08:48:10 2015 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Jun 26 08:48:10 2015 -0700
----------------------------------------------------------------------
.../MetricsRegionServerWrapperImpl.java | 11 ++--
.../wal/BoundedRegionGroupingProvider.java | 57 +++++++++++++++++++-
.../hadoop/hbase/wal/DefaultWALProvider.java | 2 +-
3 files changed, 63 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0426a5c7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index a2e02af..1bb01da 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.CacheStats;
+import org.apache.hadoop.hbase.wal.BoundedRegionGroupingProvider;
import org.apache.hadoop.hbase.wal.DefaultWALProvider;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -503,14 +504,16 @@ class MetricsRegionServerWrapperImpl
//If we've time traveled keep the last requests per second.
if ((currentTime - lastRan) > 0) {
long currentRequestCount = getTotalRequestCount();
- requestsPerSecond = (currentRequestCount - lastRequestCount) / ((currentTime - lastRan) / 1000.0);
+ requestsPerSecond = (currentRequestCount - lastRequestCount) /
+ ((currentTime - lastRan) / 1000.0);
lastRequestCount = currentRequestCount;
}
lastRan = currentTime;
- numWALFiles = DefaultWALProvider.getNumLogFiles(regionServer.walFactory);
- walFileSize = DefaultWALProvider.getLogFileSize(regionServer.walFactory);
-
+ numWALFiles = DefaultWALProvider.getNumLogFiles(regionServer.walFactory) +
+ BoundedRegionGroupingProvider.getNumLogFiles(regionServer.walFactory);
+ walFileSize = DefaultWALProvider.getLogFileSize(regionServer.walFactory) +
+ BoundedRegionGroupingProvider.getLogFileSize(regionServer.walFactory);
//Copy over computed values so that no thread sees half computed values.
numStores = tempNumStores;
numStoreFiles = tempNumStoreFiles;
http://git-wip-us.apache.org/repos/asf/hbase/blob/0426a5c7/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
index 478d5c3..e1417b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
@@ -26,7 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-
+import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
// imports for classes still in regionserver.wal
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
@@ -37,7 +37,7 @@ import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
* {@link RegionGroupingProvider}.
*/
@InterfaceAudience.Private
-class BoundedRegionGroupingProvider extends RegionGroupingProvider {
+public class BoundedRegionGroupingProvider extends RegionGroupingProvider {
private static final Log LOG = LogFactory.getLog(BoundedRegionGroupingProvider.class);
static final String NUM_REGION_GROUPS = "hbase.wal.regiongrouping.numgroups";
@@ -103,4 +103,57 @@ class BoundedRegionGroupingProvider extends RegionGroupingProvider {
throw failure;
}
}
+
+ /**
+ * iff the given WALFactory is using the BoundedRegionGroupingProvider for meta and/or non-meta,
+ * count the number of files (rolled and active). if either of them isn't, count 0
+ * for that provider.
+ * @param walFactory may not be null.
+ */
+ public static long getNumLogFiles(WALFactory walFactory) {
+ long result = 0;
+ if (walFactory.provider instanceof BoundedRegionGroupingProvider) {
+ BoundedRegionGroupingProvider groupProviders =
+ (BoundedRegionGroupingProvider)walFactory.provider;
+ for (int i = 0; i < groupProviders.delegates.length; i++) {
+ result +=
+ ((FSHLog)((DefaultWALProvider)(groupProviders.delegates[i])).log).getNumLogFiles();
+ }
+ }
+ WALProvider meta = walFactory.metaProvider.get();
+ if (meta instanceof BoundedRegionGroupingProvider) {
+ for (int i = 0; i < ((BoundedRegionGroupingProvider)meta).delegates.length; i++) {
+ result += ((FSHLog)
+ ((DefaultWALProvider)(((BoundedRegionGroupingProvider)meta).delegates[i])).log)
+ .getNumLogFiles(); }
+ }
+ return result;
+ }
+
+ /**
+ * iff the given WALFactory is using the BoundedRegionGroupingProvider for meta and/or non-meta,
+ * count the size of files (rolled and active). if either of them isn't, count 0
+ * for that provider.
+ * @param walFactory may not be null.
+ */
+ public static long getLogFileSize(WALFactory walFactory) {
+ long result = 0;
+ if (walFactory.provider instanceof BoundedRegionGroupingProvider) {
+ BoundedRegionGroupingProvider groupProviders =
+ (BoundedRegionGroupingProvider)walFactory.provider;
+ for (int i = 0; i < groupProviders.delegates.length; i++) {
+ result +=
+ ((FSHLog)((DefaultWALProvider)(groupProviders.delegates[i])).log).getLogFileSize();
+ }
+ }
+ WALProvider meta = walFactory.metaProvider.get();
+ if (meta instanceof BoundedRegionGroupingProvider) {
+ for (int i = 0; i < ((BoundedRegionGroupingProvider)meta).delegates.length; i++) {
+ result += ((FSHLog)
+ ((DefaultWALProvider)(((BoundedRegionGroupingProvider)meta).delegates[i])).log)
+ .getLogFileSize();
+ }
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0426a5c7/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
index 8d153e4..44a033b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
@@ -75,7 +75,7 @@ public class DefaultWALProvider implements WALProvider {
void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
}
- private FSHLog log = null;
+ protected FSHLog log = null;
/**
* @param factory factory that made us, identity used for FS layout. may not be null