You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/03/16 00:23:12 UTC
hbase git commit: HBASE-15135 Add metrics for storefile age
Repository: hbase
Updated Branches:
refs/heads/0.98 b248751b7 -> f0ddd3c06
HBASE-15135 Add metrics for storefile age
Conflicts:
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
Amending-Author: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f0ddd3c0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f0ddd3c0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f0ddd3c0
Branch: refs/heads/0.98
Commit: f0ddd3c066f10aa05e07433d91ee3d356658ec75
Parents: b248751
Author: Mikhail Antonov <an...@apache.org>
Authored: Mon Feb 22 02:16:40 2016 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Tue Mar 15 16:19:38 2016 -0700
----------------------------------------------------------------------
.../regionserver/MetricsRegionServerSource.java | 8 ++
.../MetricsRegionServerWrapper.java | 20 +++++
.../regionserver/MetricsRegionWrapper.java | 20 +++++
.../MetricsRegionServerSourceImpl.java | 4 +
.../regionserver/MetricsRegionSourceImpl.java | 12 +++
.../TestMetricsRegionSourceImpl.java | 23 +++++-
.../MetricsRegionServerSourceImpl.java | 8 ++
.../regionserver/MetricsRegionSourceImpl.java | 13 ++-
.../TestMetricsRegionSourceImpl.java | 20 +++++
.../hadoop/hbase/regionserver/HStore.java | 83 ++++++++++++++++++++
.../MetricsRegionServerWrapperImpl.java | 53 +++++++++++++
.../regionserver/MetricsRegionWrapperImpl.java | 55 ++++++++++++-
.../apache/hadoop/hbase/regionserver/Store.java | 25 ++++++
.../hadoop/hbase/regionserver/StoreFile.java | 7 ++
.../hbase/regionserver/StoreFileInfo.java | 11 +++
.../MetricsRegionServerWrapperStub.java | 20 +++++
.../regionserver/MetricsRegionWrapperStub.java | 20 +++++
.../hbase/regionserver/TestMetricsRegion.java | 24 +++++-
.../regionserver/TestMetricsRegionServer.java | 4 +
.../regionserver/TestRegionServerMetrics.java | 23 ++++++
20 files changed, 445 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index cc88e05..7233e0e 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -155,6 +155,14 @@ public interface MetricsRegionServerSource extends BaseSource {
String MEMSTORE_SIZE = "memStoreSize";
String MEMSTORE_SIZE_DESC = "Size of the memstore";
String STOREFILE_SIZE = "storeFileSize";
+ String MAX_STORE_FILE_AGE = "maxStoreFileAge";
+ String MIN_STORE_FILE_AGE = "minStoreFileAge";
+ String AVG_STORE_FILE_AGE = "avgStoreFileAge";
+ String NUM_REFERENCE_FILES = "numReferenceFiles";
+ String MAX_STORE_FILE_AGE_DESC = "Max age of store files hosted on this region server";
+ String MIN_STORE_FILE_AGE_DESC = "Min age of store files hosted on this region server";
+ String AVG_STORE_FILE_AGE_DESC = "Average age of store files hosted on this region server";
+ String NUM_REFERENCE_FILES_DESC = "Number of reference file on this region server";
String STOREFILE_SIZE_DESC = "Size of storefiles being served.";
String TOTAL_REQUEST_COUNT = "totalRequestCount";
String TOTAL_REQUEST_COUNT_DESC =
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index fb8ff32..598b043 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -93,6 +93,26 @@ public interface MetricsRegionServerWrapper {
long getStoreFileSize();
/**
+ * @return Max age of store files hosted on this region server
+ */
+ long getMaxStoreFileAge();
+
+ /**
+ * @return Min age of store files hosted on this region server
+ */
+ long getMinStoreFileAge();
+
+ /**
+ * @return Average age of store files hosted on this region server
+ */
+ long getAvgStoreFileAge();
+
+ /**
+ * @return Number of reference files on this region server
+ */
+ long getNumReferenceFiles();
+
+ /**
* Get the number of requests per second.
*/
double getRequestsPerSecond();
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
index 7d61f81..2a3bfb7 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
@@ -74,6 +74,26 @@ public interface MetricsRegionWrapper {
long getReadRequestCount();
/**
+ * @return Max age of store files under this region
+ */
+ long getMaxStoreFileAge();
+
+ /**
+ * @return Min age of store files under this region
+ */
+ long getMinStoreFileAge();
+
+ /**
+ * @return Average age of store files under this region
+ */
+ long getAvgStoreFileAge();
+
+ /**
+ * @return Number of reference files under this region
+ */
+ long getNumReferenceFiles();
+
+ /**
* Get the total number of mutations that have been issued against this region.
*/
long getWriteRequestCount();
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 487b670..0c48976 100644
--- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -191,6 +191,10 @@ public class MetricsRegionServerSourceImpl
.addGauge(STOREFILE_COUNT, STOREFILE_COUNT_DESC, rsWrap.getNumStoreFiles())
.addGauge(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC, rsWrap.getMemstoreSize())
.addGauge(STOREFILE_SIZE, STOREFILE_SIZE_DESC, rsWrap.getStoreFileSize())
+ .addGauge(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC, rsWrap.getMaxStoreFileAge())
+ .addGauge(MIN_STORE_FILE_AGE, MIN_STORE_FILE_AGE_DESC, rsWrap.getMinStoreFileAge())
+ .addGauge(AVG_STORE_FILE_AGE, AVG_STORE_FILE_AGE_DESC, rsWrap.getAvgStoreFileAge())
+ .addGauge(NUM_REFERENCE_FILES, NUM_REFERENCE_FILES_DESC, rsWrap.getNumReferenceFiles())
.addGauge(RS_START_TIME_NAME, RS_START_TIME_DESC, rsWrap.getStartCode())
.addCounter(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC, rsWrap.getTotalRequestCount())
.addCounter(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC, rsWrap.getReadRequestsCount())
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index 730c05e7..358d287 100644
--- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -178,6 +178,18 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
mrb.addGauge(regionNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
MetricsRegionServerSource.STOREFILE_SIZE_DESC,
this.regionWrapper.getStoreFileSize());
+ mrb.addGauge(regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_AGE,
+ MetricsRegionServerSource.MAX_STORE_FILE_AGE_DESC,
+ this.regionWrapper.getMaxStoreFileAge());
+ mrb.addGauge(regionNamePrefix + MetricsRegionServerSource.MIN_STORE_FILE_AGE,
+ MetricsRegionServerSource.MIN_STORE_FILE_AGE_DESC,
+ this.regionWrapper.getMinStoreFileAge());
+ mrb.addGauge(regionNamePrefix + MetricsRegionServerSource.AVG_STORE_FILE_AGE,
+ MetricsRegionServerSource.AVG_STORE_FILE_AGE_DESC,
+ this.regionWrapper.getAvgStoreFileAge());
+ mrb.addGauge(regionNamePrefix + MetricsRegionServerSource.NUM_REFERENCE_FILES,
+ MetricsRegionServerSource.NUM_REFERENCE_FILES_DESC,
+ this.regionWrapper.getNumReferenceFiles());
mrb.addCounter(regionNamePrefix + MetricsRegionServerSource.READ_REQUEST_COUNT,
MetricsRegionServerSource.READ_REQUEST_COUNT_DESC,
this.regionWrapper.getReadRequestCount());
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
index 95c3225..aca04c7 100644
--- a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
+++ b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
@@ -18,15 +18,12 @@
package org.apache.hadoop.hbase.regionserver;
-import java.util.HashMap;
import java.util.Map;
import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
-
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.testclassification.MetricsTests;
-
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -104,6 +101,26 @@ public class TestMetricsRegionSourceImpl {
}
@Override
+ public long getMaxStoreFileAge() {
+ return 0;
+ }
+
+ @Override
+ public long getMinStoreFileAge() {
+ return 0;
+ }
+
+ @Override
+ public long getAvgStoreFileAge() {
+ return 0;
+ }
+
+ @Override
+ public long getNumReferenceFiles() {
+ return 0;
+ }
+
+ @Override
public long getReadRequestCount() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 6827b94..5c8189b 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -195,6 +195,14 @@ public class MetricsRegionServerSourceImpl
.addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles())
.addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemstoreSize())
.addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize())
+ .addGauge(Interns.info(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC),
+ rsWrap.getMaxStoreFileAge())
+ .addGauge(Interns.info(MIN_STORE_FILE_AGE, MIN_STORE_FILE_AGE_DESC),
+ rsWrap.getMinStoreFileAge())
+ .addGauge(Interns.info(AVG_STORE_FILE_AGE, AVG_STORE_FILE_AGE_DESC),
+ rsWrap.getAvgStoreFileAge())
+ .addGauge(Interns.info(NUM_REFERENCE_FILES, NUM_REFERENCE_FILES_DESC),
+ rsWrap.getNumReferenceFiles())
.addGauge(Interns.info(RS_START_TIME_NAME, RS_START_TIME_DESC),
rsWrap.getStartCode())
.addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC),
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index 1bad5b3..547524e 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -183,6 +183,18 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
MetricsRegionServerSource.STOREFILE_SIZE_DESC),
this.regionWrapper.getStoreFileSize());
+ mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_AGE,
+ MetricsRegionServerSource.MAX_STORE_FILE_AGE_DESC),
+ this.regionWrapper.getMaxStoreFileAge());
+ mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.MIN_STORE_FILE_AGE,
+ MetricsRegionServerSource.MIN_STORE_FILE_AGE_DESC),
+ this.regionWrapper.getMinStoreFileAge());
+ mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.AVG_STORE_FILE_AGE,
+ MetricsRegionServerSource.AVG_STORE_FILE_AGE_DESC),
+ this.regionWrapper.getAvgStoreFileAge());
+ mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.NUM_REFERENCE_FILES,
+ MetricsRegionServerSource.NUM_REFERENCE_FILES_DESC),
+ this.regionWrapper.getNumReferenceFiles());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.COMPACTIONS_COMPLETED_COUNT,
MetricsRegionSource.COMPACTIONS_COMPLETED_DESC),
this.regionWrapper.getNumCompactionsCompleted());
@@ -218,6 +230,5 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
MetricsRegionSource.COPROCESSOR_EXECUTION_STATISTICS_DESC + "99th percentile: "), ds
.getPercentile(99d) / 1000);
}
-
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
index a7339f2..539e235 100644
--- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
@@ -106,6 +106,26 @@ public class TestMetricsRegionSourceImpl {
}
@Override
+ public long getMaxStoreFileAge() {
+ return 0;
+ }
+
+ @Override
+ public long getMinStoreFileAge() {
+ return 0;
+ }
+
+ @Override
+ public long getAvgStoreFileAge() {
+ return 0;
+ }
+
+ @Override
+ public long getNumReferenceFiles() {
+ return 0;
+ }
+
+ @Override
public long getWriteRequestCount() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 1f462a9..9c9c1da 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -1966,6 +1966,89 @@ public class HStore implements Store {
}
@Override
+ public long getMaxStoreFileAge() {
+ long earliestTS = Long.MAX_VALUE;
+ for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
+ StoreFile.Reader r = s.getReader();
+ if (r == null) {
+ LOG.warn("StoreFile " + s + " has a null Reader");
+ continue;
+ }
+ if (!s.isHFile()) {
+ continue;
+ }
+ long createdTS = s.getFileInfo().getCreatedTimestamp();
+ earliestTS = (createdTS < earliestTS) ? createdTS : earliestTS;
+ }
+ long now = EnvironmentEdgeManager.currentTimeMillis();
+ return now - earliestTS;
+ }
+
+ @Override
+ public long getMinStoreFileAge() {
+ long latestTS = 0;
+ for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
+ StoreFile.Reader r = s.getReader();
+ if (r == null) {
+ LOG.warn("StoreFile " + s + " has a null Reader");
+ continue;
+ }
+ if (!s.isHFile()) {
+ continue;
+ }
+ long createdTS = s.getFileInfo().getCreatedTimestamp();
+ latestTS = (createdTS > latestTS) ? createdTS : latestTS;
+ }
+ long now = EnvironmentEdgeManager.currentTimeMillis();
+ return now - latestTS;
+ }
+
+ @Override
+ public long getAvgStoreFileAge() {
+ long sum = 0, count = 0;
+ for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
+ StoreFile.Reader r = s.getReader();
+ if (r == null) {
+ LOG.warn("StoreFile " + s + " has a null Reader");
+ continue;
+ }
+ if (!s.isHFile()) {
+ continue;
+ }
+ sum += s.getFileInfo().getCreatedTimestamp();
+ count++;
+ }
+ if (count == 0) {
+ return 0;
+ }
+ long avgTS = sum / count;
+ long now = EnvironmentEdgeManager.currentTimeMillis();
+ return now - avgTS;
+ }
+
+ @Override
+ public long getNumReferenceFiles() {
+ long numRefFiles = 0;
+ for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
+ if (s.isReference()) {
+ numRefFiles++;
+ }
+ }
+ return numRefFiles;
+ }
+
+ @Override
+ public long getNumHFiles() {
+ long numHFiles = 0;
+ for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
+ if (s.isHFile()) {
+ numHFiles++;
+ }
+ }
+ return numHFiles;
+ }
+
+ @Override
public long getStoreSizeUncompressed() {
return this.totalUncompressedBytes;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index aaa318e..98d80fc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -55,6 +55,10 @@ class MetricsRegionServerWrapperImpl
private volatile long numStoreFiles = 0;
private volatile long memstoreSize = 0;
private volatile long storeFileSize = 0;
+ private volatile long maxStoreFileAge = 0;
+ private volatile long minStoreFileAge = 0;
+ private volatile long avgStoreFileAge = 0;
+ private volatile long numReferenceFiles = 0;
private volatile double requestsPerSecond = 0.0;
private volatile long readRequestsCount = 0;
private volatile long writeRequestsCount = 0;
@@ -302,6 +306,26 @@ class MetricsRegionServerWrapperImpl
}
@Override
+ public long getMaxStoreFileAge() {
+ return maxStoreFileAge;
+ }
+
+ @Override
+ public long getMinStoreFileAge() {
+ return minStoreFileAge;
+ }
+
+ @Override
+ public long getAvgStoreFileAge() {
+ return avgStoreFileAge;
+ }
+
+ @Override
+ public long getNumReferenceFiles() {
+ return numReferenceFiles;
+ }
+
+ @Override
public long getMemstoreSize() {
return memstoreSize;
}
@@ -425,6 +449,11 @@ class MetricsRegionServerWrapperImpl
long tempNumStoreFiles = 0;
long tempMemstoreSize = 0;
long tempStoreFileSize = 0;
+ long tempMaxStoreFileAge = 0;
+ long tempNumReferenceFiles = 0;
+ long avgAgeNumerator = 0;
+ long numHFiles = 0;
+ long tempMinStoreFileAge = Long.MAX_VALUE;
long tempReadRequestsCount = 0;
long tempWriteRequestsCount = 0;
long tempCheckAndMutateChecksFailed = 0;
@@ -456,6 +485,20 @@ class MetricsRegionServerWrapperImpl
tempNumStoreFiles += store.getStorefilesCount();
tempMemstoreSize += store.getMemStoreSize();
tempStoreFileSize += store.getStorefilesSize();
+
+ long storeMaxStoreFileAge = store.getMaxStoreFileAge();
+ tempMaxStoreFileAge = (storeMaxStoreFileAge > tempMaxStoreFileAge) ?
+ storeMaxStoreFileAge : tempMaxStoreFileAge;
+
+ long storeMinStoreFileAge = store.getMinStoreFileAge();
+ tempMinStoreFileAge = (storeMinStoreFileAge < tempMinStoreFileAge) ?
+ storeMinStoreFileAge : tempMinStoreFileAge;
+
+ long storeHFiles = store.getNumHFiles();
+ avgAgeNumerator += store.getAvgStoreFileAge() * storeHFiles;
+ numHFiles += storeHFiles;
+ tempNumReferenceFiles += store.getNumReferenceFiles();
+
tempStorefileIndexSize += store.getStorefilesIndexSize();
tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
@@ -511,6 +554,16 @@ class MetricsRegionServerWrapperImpl
numStoreFiles = tempNumStoreFiles;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
+ maxStoreFileAge = tempMaxStoreFileAge;
+ if (tempMinStoreFileAge != Long.MAX_VALUE) {
+ minStoreFileAge = tempMinStoreFileAge;
+ }
+
+ if (numHFiles != 0) {
+ avgStoreFileAge = avgAgeNumerator / numHFiles;
+ }
+
+ numReferenceFiles= tempNumReferenceFiles;
readRequestsCount = tempReadRequestsCount;
writeRequestsCount = tempWriteRequestsCount;
checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed;
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
index be05f3e..1cdcb0f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
@@ -46,6 +46,10 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
private long memstoreSize;
private long storeFileSize;
private Map<String, DescriptiveStatistics> coprocessorTimes;
+ private long maxStoreFileAge;
+ private long minStoreFileAge;
+ private long avgStoreFileAge;
+ private long numReferenceFiles;
private ScheduledFuture<?> regionMetricsUpdateTask;
@@ -135,6 +139,26 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
return this.region.compactionsFinished.get();
}
+ @Override
+ public long getMaxStoreFileAge() {
+ return maxStoreFileAge;
+ }
+
+ @Override
+ public long getMinStoreFileAge() {
+ return minStoreFileAge;
+ }
+
+ @Override
+ public long getAvgStoreFileAge() {
+ return avgStoreFileAge;
+ }
+
+ @Override
+ public long getNumReferenceFiles() {
+ return numReferenceFiles;
+ }
+
public class HRegionMetricsWrapperRunnable implements Runnable {
@Override
@@ -142,20 +166,49 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
long tempNumStoreFiles = 0;
long tempMemstoreSize = 0;
long tempStoreFileSize = 0;
+ long tempMaxStoreFileAge = 0;
+ long tempMinStoreFileAge = Long.MAX_VALUE;
+ long tempNumReferenceFiles = 0;
+ long avgAgeNumerator = 0;
+ long numHFiles = 0;
if (region.stores != null) {
for (Store store : region.stores.values()) {
tempNumStoreFiles += store.getStorefilesCount();
tempMemstoreSize += store.getMemStoreSize();
tempStoreFileSize += store.getStorefilesSize();
+
+ long storeMaxStoreFileAge = store.getMaxStoreFileAge();
+ tempMaxStoreFileAge = (storeMaxStoreFileAge > tempMaxStoreFileAge) ?
+ storeMaxStoreFileAge : tempMaxStoreFileAge;
+
+ long storeMinStoreFileAge = store.getMinStoreFileAge();
+ tempMinStoreFileAge = (storeMinStoreFileAge < tempMinStoreFileAge) ?
+ storeMinStoreFileAge : tempMinStoreFileAge;
+
+ long storeHFiles = store.getNumHFiles();
+ avgAgeNumerator += store.getAvgStoreFileAge() * storeHFiles;
+ numHFiles += storeHFiles;
+ tempNumReferenceFiles += store.getNumReferenceFiles();
}
}
numStoreFiles = tempNumStoreFiles;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
- coprocessorTimes = region.getCoprocessorHost().getCoprocessorExecutionStatistics();
+ maxStoreFileAge = tempMaxStoreFileAge;
+ if (tempMinStoreFileAge != Long.MAX_VALUE) {
+ minStoreFileAge = tempMinStoreFileAge;
+ }
+
+ if (numHFiles != 0) {
+ avgStoreFileAge = avgAgeNumerator / numHFiles;
+ }
+
+ numReferenceFiles = tempNumReferenceFiles;
+
+ coprocessorTimes = region.getCoprocessorHost().getCoprocessorExecutionStatistics();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 532208a..a110ad3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -324,6 +324,31 @@ public interface Store extends HeapSize, StoreConfigInformation {
int getStorefilesCount();
/**
+ * @return Max age of store files in this store
+ */
+ long getMaxStoreFileAge();
+
+ /**
+ * @return Min age of store files in this store
+ */
+ long getMinStoreFileAge();
+
+ /**
+ * @return Average age of store files in this store, 0 if no store files
+ */
+ long getAvgStoreFileAge();
+
+ /**
+ * @return Number of reference files in this store
+ */
+ long getNumReferenceFiles();
+
+ /**
+ * @return Number of HFiles in this store
+ */
+ long getNumHFiles();
+
+ /**
* @return The size of the store files, in bytes, uncompressed.
*/
long getStoreSizeUncompressed();
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 5ba7140..f222379 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -261,6 +261,13 @@ public class StoreFile {
}
/**
+ * @return True if this is HFile.
+ */
+ public boolean isHFile() {
+ return this.fileInfo.isHFile(this.fileInfo.getPath());
+ }
+
+ /**
* @return True if this file was made by a major compaction.
*/
public boolean isMajorCompaction() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
index 5e46d94..f8d6dc9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
@@ -83,6 +83,9 @@ public class StoreFileInfo {
private RegionCoprocessorHost coprocessorHost;
+ // timestamp on when the file was created, is 0 and ignored for reference or link files
+ private long createdTimestamp;
+
/**
* Create a Store File Info
* @param conf the {@link Configuration} to use
@@ -124,6 +127,7 @@ public class StoreFileInfo {
" reference to " + referencePath);
} else if (isHFile(p)) {
// HFile
+ this.createdTimestamp = fileStatus.getModificationTime();
this.reference = null;
this.link = null;
} else {
@@ -311,6 +315,13 @@ public class StoreFileInfo {
return m.matches() && m.groupCount() > 1;
}
+ /**
+ * @return timestamp when this file was created (as returned by filesystem)
+ */
+ public long getCreatedTimestamp() {
+ return createdTimestamp;
+ }
+
/*
* Return path to the file referred to by a Reference. Presumes a directory
* hierarchy of <code>${hbase.rootdir}/data/${namespace}/tablename/regionname/familyname</code>.
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 10159f1..0b4bbb1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -71,6 +71,26 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
}
@Override
+ public long getMaxStoreFileAge() {
+ return 2;
+ }
+
+ @Override
+ public long getMinStoreFileAge() {
+ return 2;
+ }
+
+ @Override
+ public long getAvgStoreFileAge() {
+ return 2;
+ }
+
+ @Override
+ public long getNumReferenceFiles() {
+ return 2;
+ }
+
+ @Override
public double getRequestsPerSecond() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
index 2658c0a..c41f93a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
@@ -66,6 +66,26 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
}
@Override
+ public long getMaxStoreFileAge() {
+ return 2;
+ }
+
+ @Override
+ public long getMinStoreFileAge() {
+ return 2;
+ }
+
+ @Override
+ public long getAvgStoreFileAge() {
+ return 2;
+ }
+
+ @Override
+ public long getNumReferenceFiles() {
+ return 2;
+ }
+
+ @Override
public long getWriteRequestCount() {
return 106;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
index 8963a34..de305e1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
@@ -35,9 +35,27 @@ public class TestMetricsRegion {
MetricsRegion mr = new MetricsRegion(new MetricsRegionWrapperStub());
MetricsRegionAggregateSource agg = mr.getSource().getAggregateSource();
- HELPER.assertGauge("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_storeCount", 101, agg);
- HELPER.assertGauge("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_storeFileCount", 102, agg);
- HELPER.assertGauge("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_memstoreSize", 103, agg);
+ HELPER.assertGauge(
+ "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_storeCount",
+ 101, agg);
+ HELPER.assertGauge(
+ "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_storeFileCount",
+ 102, agg);
+ HELPER.assertGauge(
+ "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_memstoreSize",
+ 103, agg);
+ HELPER.assertGauge(
+ "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_maxStoreFileAge",
+ 2, agg);
+ HELPER.assertGauge(
+ "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_minStoreFileAge",
+ 2, agg);
+ HELPER.assertGauge(
+ "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_avgStoreFileAge",
+ 2, agg);
+ HELPER.assertGauge(
+ "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_numReferenceFiles",
+ 2, agg);
mr.close();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
index 90dde70..dff0db8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
@@ -59,6 +59,10 @@ public class TestMetricsRegionServer {
HELPER.assertGauge("regionServerStartTime", 100, serverSource);
HELPER.assertGauge("regionCount", 101, serverSource);
HELPER.assertGauge("storeCount", 2, serverSource);
+ HELPER.assertGauge("maxStoreFileAge", 2, serverSource);
+ HELPER.assertGauge("minStoreFileAge", 2, serverSource);
+ HELPER.assertGauge("avgStoreFileAge", 2, serverSource);
+ HELPER.assertGauge("numReferenceFiles", 2, serverSource);
HELPER.assertGauge("hlogFileCount", 10, serverSource);
HELPER.assertGauge("hlogFileSize", 1024000, serverSource);
HELPER.assertGauge("storeFileCount", 300, serverSource);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f0ddd3c0/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 33c0f59..b66c331 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -249,6 +249,29 @@ public class TestRegionServerMetrics {
}
@Test
+ public void testStoreFileAge() throws Exception {
+ TableName tableName = TableName.valueOf("testStoreFileAge");
+ byte[] cf = Bytes.toBytes("d");
+ byte[] row = Bytes.toBytes("rk");
+ byte[] qualifier = Bytes.toBytes("qual");
+ byte[] val = Bytes.toBytes("Value");
+
+ //Force a hfile.
+ HTable t = TEST_UTIL.createTable(tableName, cf);
+ Put p = new Put(row);
+ p.add(cf, qualifier, val);
+ t.put(p);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+
+ metricsRegionServer.getRegionServerWrapper().forceRecompute();
+ assertTrue(metricsHelper.getGaugeLong("maxStoreFileAge", serverSource) > 0);
+ assertTrue(metricsHelper.getGaugeLong("minStoreFileAge", serverSource) > 0);
+ assertTrue(metricsHelper.getGaugeLong("avgStoreFileAge", serverSource) > 0);
+
+ t.close();
+ }
+
+ @Test
public void testCheckAndPutCount() throws Exception {
String tableNameString = "testCheckAndPutCount";
byte[] tableName = Bytes.toBytes(tableNameString);