You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2019/05/30 22:49:58 UTC
[hbase] branch branch-2 updated: HBASE-22459 Expose store reader
reference count (#248)
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new d06c3af HBASE-22459 Expose store reader reference count (#248)
d06c3af is described below
commit d06c3af9806c414506243552a476a50c74fa5ecc
Author: Andrew Purtell <ap...@apache.org>
AuthorDate: Thu May 30 15:04:16 2019 -0700
HBASE-22459 Expose store reader reference count (#248)
---
.../main/java/org/apache/hadoop/hbase/RegionLoad.java | 8 ++++++++
.../java/org/apache/hadoop/hbase/RegionMetrics.java | 4 ++++
.../org/apache/hadoop/hbase/RegionMetricsBuilder.java | 18 ++++++++++++++++++
.../hbase/regionserver/MetricsRegionServerSource.java | 2 ++
.../hbase/regionserver/MetricsRegionWrapper.java | 5 +++++
.../hbase/regionserver/MetricsRegionSourceImpl.java | 4 ++++
.../regionserver/TestMetricsRegionSourceImpl.java | 5 +++++
.../src/main/protobuf/ClusterStatus.proto | 6 ++++++
hbase-protocol/src/main/protobuf/ClusterStatus.proto | 6 ++++++
.../org/apache/hadoop/hbase/regionserver/HStore.java | 6 ++++++
.../hbase/regionserver/MetricsRegionWrapperImpl.java | 11 ++++++++++-
.../hbase/regionserver/MetricsRegionWrapperStub.java | 5 +++++
12 files changed, 79 insertions(+), 1 deletion(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
index 242e5ea..41c68ef 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
@@ -359,6 +359,13 @@ public class RegionLoad implements RegionMetrics {
}
/**
+ * @return the reference count for the stores of this region
+ */
+ public int getStoreRefCount() {
+ return metrics.getStoreRefCount();
+ }
+
+ /**
* @see java.lang.Object#toString()
*/
@Override
@@ -366,6 +373,7 @@ public class RegionLoad implements RegionMetrics {
StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "numberOfStores",
this.getStores());
Strings.appendKeyValue(sb, "numberOfStorefiles", this.getStorefiles());
+ Strings.appendKeyValue(sb, "storeRefCount", this.getStoreRefCount());
Strings.appendKeyValue(sb, "storefileUncompressedSizeMB",
this.getStoreUncompressedSizeMB());
Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp",
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
index e73683f..9de98f3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
@@ -144,4 +144,8 @@ public interface RegionMetrics {
*/
long getLastMajorCompactionTimestamp();
+ /**
+ * @return the reference count for the stores of this region
+ */
+ int getStoreRefCount();
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
index f2a549a..77c9f7a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
@@ -64,6 +64,7 @@ public final class RegionMetricsBuilder {
Size.Unit.KILOBYTE))
.setStoreCount(regionLoadPB.getStores())
.setStoreFileCount(regionLoadPB.getStorefiles())
+ .setStoreRefCount(regionLoadPB.getStoreRefCount())
.setStoreFileSize(new Size(regionLoadPB.getStorefileSizeMB(), Size.Unit.MEGABYTE))
.setStoreSequenceIds(regionLoadPB.getStoreCompleteSequenceIdList().stream()
.collect(Collectors.toMap(
@@ -109,6 +110,7 @@ public final class RegionMetricsBuilder {
.get(Size.Unit.KILOBYTE))
.setStores(regionMetrics.getStoreCount())
.setStorefiles(regionMetrics.getStoreCount())
+ .setStoreRefCount(regionMetrics.getStoreRefCount())
.setStorefileSizeMB((int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE))
.addAllStoreCompleteSequenceId(toStoreSequenceId(regionMetrics.getStoreSequenceId()))
.setStoreUncompressedSizeMB(
@@ -123,6 +125,7 @@ public final class RegionMetricsBuilder {
private final byte[] name;
private int storeCount;
private int storeFileCount;
+ private int storeRefCount;
private long compactingCellCount;
private long compactedCellCount;
private Size storeFileSize = Size.ZERO;
@@ -151,6 +154,10 @@ public final class RegionMetricsBuilder {
this.storeFileCount = value;
return this;
}
+ public RegionMetricsBuilder setStoreRefCount(int value) {
+ this.storeRefCount = value;
+ return this;
+ }
public RegionMetricsBuilder setCompactingCellCount(long value) {
this.compactingCellCount = value;
return this;
@@ -220,6 +227,7 @@ public final class RegionMetricsBuilder {
return new RegionMetricsImpl(name,
storeCount,
storeFileCount,
+ storeRefCount,
compactingCellCount,
compactedCellCount,
storeFileSize,
@@ -242,6 +250,7 @@ public final class RegionMetricsBuilder {
private final byte[] name;
private final int storeCount;
private final int storeFileCount;
+ private final int storeRefCount;
private final long compactingCellCount;
private final long compactedCellCount;
private final Size storeFileSize;
@@ -261,6 +270,7 @@ public final class RegionMetricsBuilder {
RegionMetricsImpl(byte[] name,
int storeCount,
int storeFileCount,
+ int storeRefCount,
final long compactingCellCount,
long compactedCellCount,
Size storeFileSize,
@@ -280,6 +290,7 @@ public final class RegionMetricsBuilder {
this.name = Preconditions.checkNotNull(name);
this.storeCount = storeCount;
this.storeFileCount = storeFileCount;
+ this.storeRefCount = storeRefCount;
this.compactingCellCount = compactingCellCount;
this.compactedCellCount = compactedCellCount;
this.storeFileSize = Preconditions.checkNotNull(storeFileSize);
@@ -314,6 +325,11 @@ public final class RegionMetricsBuilder {
}
@Override
+ public int getStoreRefCount() {
+ return storeRefCount;
+ }
+
+ @Override
public Size getStoreFileSize() {
return storeFileSize;
}
@@ -399,6 +415,8 @@ public final class RegionMetricsBuilder {
this.getStoreCount());
Strings.appendKeyValue(sb, "storeFileCount",
this.getStoreFileCount());
+ Strings.appendKeyValue(sb, "storeRefCount",
+ this.getStoreRefCount());
Strings.appendKeyValue(sb, "uncompressedStoreFileSize",
this.getUncompressedStoreFileSize());
Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp",
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index fc9cde7..218c955 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -231,6 +231,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String WALFILE_SIZE_DESC = "Size of all WAL Files";
String STOREFILE_COUNT = "storeFileCount";
String STOREFILE_COUNT_DESC = "Number of Store Files";
+ String STORE_REF_COUNT = "storeRefCount";
+ String STORE_REF_COUNT_DESC = "Store reference count";
String MEMSTORE_SIZE = "memStoreSize";
String MEMSTORE_SIZE_DESC = "Size of the memstore";
String STOREFILE_SIZE = "storeFileSize";
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
index 5e98c61..4a43839 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
@@ -152,4 +152,9 @@ public interface MetricsRegionWrapper {
* Get the replica id of this region.
*/
int getReplicaId();
+
+ /**
+ * @return the number of references active on the store
+ */
+ long getStoreRefCount();
}
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index ff1f42d..8a3ec17 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -214,6 +214,10 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
MetricsRegionServerSource.STOREFILE_COUNT_DESC),
this.regionWrapper.getNumStoreFiles());
mrb.addGauge(Interns.info(
+ regionNamePrefix + MetricsRegionServerSource.STORE_REF_COUNT,
+ MetricsRegionServerSource.STORE_REF_COUNT),
+ this.regionWrapper.getStoreRefCount());
+ mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE,
MetricsRegionServerSource.MEMSTORE_SIZE_DESC),
this.regionWrapper.getMemStoreSize());
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
index 82bfc0b..4ec3b69 100644
--- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
@@ -95,6 +95,11 @@ public class TestMetricsRegionSourceImpl {
}
@Override
+ public long getStoreRefCount() {
+ return 0;
+ }
+
+ @Override
public long getMemStoreSize() {
return 0;
}
diff --git a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
index 563db9f..d1487af 100644
--- a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
@@ -143,6 +143,12 @@ message RegionLoad {
/** the current total filtered read requests made to region */
optional uint64 filtered_read_requests_count = 19;
+
+ /** master defines cp_requests_count = 20, the current total coprocessor
+ requests made to region */
+
+ /** the number of references active on the store */
+ optional int32 store_ref_count = 21 [default = 0];
}
/* Server-level protobufs */
diff --git a/hbase-protocol/src/main/protobuf/ClusterStatus.proto b/hbase-protocol/src/main/protobuf/ClusterStatus.proto
index b172db5..30fd409 100644
--- a/hbase-protocol/src/main/protobuf/ClusterStatus.proto
+++ b/hbase-protocol/src/main/protobuf/ClusterStatus.proto
@@ -139,6 +139,12 @@ message RegionLoad {
/** the current total filtered read requests made to region */
optional uint64 filtered_read_requests_count = 19;
+
+ /** master defines cp_requests_count = 20, the current total coprocessor
+ requests made to region */
+
+ /** the number of references active on the store */
+ optional int32 store_ref_count = 21 [default = 0];
}
/* Server-level protobufs */
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 86b6c1e..856a405 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -2668,4 +2668,10 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
public int getCurrentParallelPutCount() {
return currentParallelPutCount.get();
}
+
+ public int getStoreRefCount() {
+ return this.storeEngine.getStoreFileManager().getStorefiles().stream()
+ .filter(sf -> sf.getReader() != null).filter(HStoreFile::isHFile)
+ .mapToInt(HStoreFile::getRefCount).sum();
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
index c073ef0..b793588 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
@@ -48,6 +48,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
private ScheduledExecutorService executor;
private Runnable runnable;
private long numStoreFiles;
+ private long storeRefCount;
private long memstoreSize;
private long storeFileSize;
private long maxStoreFileAge;
@@ -120,6 +121,11 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
}
@Override
+ public long getStoreRefCount() {
+ return storeRefCount;
+ }
+
+ @Override
public long getReadRequestCount() {
return this.region.getReadRequestsCount();
}
@@ -221,6 +227,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
@Override
public void run() {
long tempNumStoreFiles = 0;
+ int tempStoreRefCount = 0;
long tempMemstoreSize = 0;
long tempStoreFileSize = 0;
long tempMaxStoreFileAge = 0;
@@ -232,8 +239,9 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
long avgAgeNumerator = 0;
long numHFiles = 0;
if (region.stores != null) {
- for (Store store : region.stores.values()) {
+ for (HStore store : region.stores.values()) {
tempNumStoreFiles += store.getStorefilesCount();
+ tempStoreRefCount += store.getStoreRefCount();
tempMemstoreSize += store.getMemStoreSize().getDataSize();
tempStoreFileSize += store.getStorefilesSize();
OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@@ -260,6 +268,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
}
numStoreFiles = tempNumStoreFiles;
+ storeRefCount = tempStoreRefCount;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
index 1b1d0bf..ad04cde 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
@@ -61,6 +61,11 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
}
@Override
+ public long getStoreRefCount() {
+ return 0;
+ }
+
+ @Override
public long getMemStoreSize() {
return 103;
}