You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2016/11/01 19:49:28 UTC
[04/50] [abbrv] hbase git commit: HBASE-16661 Add last major
compaction age to per-region metrics
HBASE-16661 Add last major compaction age to per-region metrics
Signed-off-by: Gary Helmling <ga...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/59ca4dad
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/59ca4dad
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/59ca4dad
Branch: refs/heads/branch-1
Commit: 59ca4dad70cee46314c992766fd9303d1e41ee2c
Parents: 66038b8
Author: Dustin Pho <ph...@fb.com>
Authored: Sat Sep 24 14:58:37 2016 -0700
Committer: Gary Helmling <ga...@apache.org>
Committed: Mon Oct 10 15:21:53 2016 -0700
----------------------------------------------------------------------
.../hbase/regionserver/MetricsRegionSource.java | 2 ++
.../hbase/regionserver/MetricsRegionWrapper.java | 5 +++++
.../regionserver/MetricsRegionSourceImpl.java | 4 ++++
.../regionserver/TestMetricsRegionSourceImpl.java | 5 +++++
.../apache/hadoop/hbase/regionserver/HRegion.java | 9 +++------
.../regionserver/MetricsRegionWrapperImpl.java | 17 +++++++++++++++++
.../apache/hadoop/hbase/regionserver/Region.java | 4 ++--
.../regionserver/MetricsRegionWrapperStub.java | 5 +++++
8 files changed, 43 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/59ca4dad/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
index 911c757..12ef07c 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
@@ -29,10 +29,12 @@ public interface MetricsRegionSource extends Comparable<MetricsRegionSource> {
String SIZE_VALUE_NAME = "size";
String COMPACTIONS_COMPLETED_COUNT = "compactionsCompletedCount";
String COMPACTIONS_FAILED_COUNT = "compactionsFailedCount";
+ String LAST_MAJOR_COMPACTION_AGE = "lastMajorCompactionAge";
String NUM_BYTES_COMPACTED_COUNT = "numBytesCompactedCount";
String NUM_FILES_COMPACTED_COUNT = "numFilesCompactedCount";
String COMPACTIONS_COMPLETED_DESC = "Number of compactions that have completed.";
String COMPACTIONS_FAILED_DESC = "Number of compactions that have failed.";
+ String LAST_MAJOR_COMPACTION_DESC = "Age of the last major compaction in milliseconds.";
String NUM_BYTES_COMPACTED_DESC =
"Sum of filesize on all files entering a finished, successful or aborted, compaction";
String NUM_FILES_COMPACTED_DESC =
http://git-wip-us.apache.org/repos/asf/hbase/blob/59ca4dad/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
index 0482d2a..9b7acd3 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
@@ -101,6 +101,11 @@ public interface MetricsRegionWrapper {
long getNumCompactionsCompleted();
/**
+ * @return Age of the last major compaction
+ */
+ long getLastMajorCompactionAge();
+
+ /**
* Returns the total number of compactions that have been reported as failed on this region.
* Note that a given compaction can be reported as both completed and failed if an exception
* is thrown in the processing after {@code HRegion.compact()}.
http://git-wip-us.apache.org/repos/asf/hbase/blob/59ca4dad/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index ae579cf..16f36bb 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -269,6 +269,10 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
MetricsRegionSource.COMPACTIONS_FAILED_DESC),
this.regionWrapper.getNumCompactionsFailed());
mrb.addCounter(Interns.info(
+ regionNamePrefix + MetricsRegionSource.LAST_MAJOR_COMPACTION_AGE,
+ MetricsRegionSource.LAST_MAJOR_COMPACTION_DESC),
+ this.regionWrapper.getLastMajorCompactionAge());
+ mrb.addCounter(Interns.info(
regionNamePrefix + MetricsRegionSource.NUM_BYTES_COMPACTED_COUNT,
MetricsRegionSource.NUM_BYTES_COMPACTED_DESC),
this.regionWrapper.getNumBytesCompacted());
http://git-wip-us.apache.org/repos/asf/hbase/blob/59ca4dad/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
index 7498a74..e1cb6b8 100644
--- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
@@ -144,6 +144,11 @@ public class TestMetricsRegionSourceImpl {
}
@Override
+ public long getLastMajorCompactionAge() {
+ return 0;
+ }
+
+ @Override
public long getNumCompactionsCompleted() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/59ca4dad/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c399f4a..c18564d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1663,7 +1663,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
@Override
- public long getOldestHfileTs(boolean majorCompactioOnly) throws IOException {
+ public long getOldestHfileTs(boolean majorCompactionOnly) throws IOException {
long result = Long.MAX_VALUE;
for (Store store : getStores()) {
Collection<StoreFile> storeFiles = store.getStorefiles();
@@ -1673,12 +1673,9 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
if (sfReader == null) continue;
HFile.Reader reader = sfReader.getHFileReader();
if (reader == null) continue;
- if (majorCompactioOnly) {
+ if (majorCompactionOnly) {
byte[] val = reader.loadFileInfo().get(StoreFile.MAJOR_COMPACTION_KEY);
- if (val == null) continue;
- if (val == null || !Bytes.toBoolean(val)) {
- continue;
- }
+ if (val == null || !Bytes.toBoolean(val)) continue;
}
result = Math.min(result, reader.getFileContext().getFileCreateTime());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/59ca4dad/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
index 46e13f7..493ee18 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
@@ -25,15 +25,20 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.metrics2.MetricsExecutor;
@InterfaceAudience.Private
public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable {
+ private static final Log LOG = LogFactory.getLog(MetricsRegionWrapperImpl.class);
+
public static final int PERIOD = 45;
public static final String UNKNOWN = "unknown";
@@ -136,6 +141,18 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
}
@Override
+ public long getLastMajorCompactionAge() {
+ long lastMajorCompactionTs = 0L;
+ try {
+ lastMajorCompactionTs = this.region.getOldestHfileTs(true);
+ } catch (IOException ioe) {
+ LOG.error("Could not load HFile info ", ioe);
+ }
+ long now = EnvironmentEdgeManager.currentTime();
+ return now - lastMajorCompactionTs;
+ }
+
+ @Override
public long getNumCompactionsFailed() {
return this.region.compactionsFailed.get();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/59ca4dad/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
index 8827967..c763ac0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
@@ -141,10 +141,10 @@ public interface Region extends ConfigurationObserver {
/**
* This can be used to determine the last time all files of this region were major compacted.
- * @param majorCompactioOnly Only consider HFile that are the result of major compaction
+ * @param majorCompactionOnly Only consider HFile that are the result of major compaction
* @return the timestamp of the oldest HFile for all stores of this region
*/
- long getOldestHfileTs(boolean majorCompactioOnly) throws IOException;
+ long getOldestHfileTs(boolean majorCompactionOnly) throws IOException;
/**
* @return map of column family names to max sequence id that was read from storage when this
http://git-wip-us.apache.org/repos/asf/hbase/blob/59ca4dad/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
index ab340d9..8ae1180 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
@@ -116,6 +116,11 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
}
@Override
+ public long getLastMajorCompactionAge() {
+ return 0;
+ }
+
+ @Override
public long getNumCompactionsFailed() {
return 0;
}