You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2023/03/29 17:43:23 UTC
[hadoop] branch trunk updated: HDFS-16959. RBF: State store cache loading metrics (#5497)
This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b4bcbb9515b HDFS-16959. RBF: State store cache loading metrics (#5497)
b4bcbb9515b is described below
commit b4bcbb9515b5b264156b379034b8e9c923bcb25d
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Wed Mar 29 10:43:13 2023 -0700
HDFS-16959. RBF: State store cache loading metrics (#5497)
---
.../hadoop-common/src/site/markdown/Metrics.md | 24 ++++++++-------
.../federation/metrics/StateStoreMetrics.java | 28 +++++++++++++++++
.../server/federation/store/CachedRecordStore.java | 2 ++
.../store/driver/TestStateStoreDriverBase.java | 36 ++++++++++++++++++++++
.../store/driver/TestStateStoreFile.java | 12 ++++++++
.../store/driver/TestStateStoreFileSystem.java | 12 ++++++++
.../federation/store/driver/TestStateStoreZK.java | 14 +++++++++
7 files changed, 117 insertions(+), 11 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index a551e3ae15f..0777fc42abe 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -592,17 +592,19 @@ StateStoreMetrics
-----------------
StateStoreMetrics shows the statistics of the State Store component in Router-based federation.
-| Name | Description |
-|:---- |:---- |
-| `ReadsNumOps` | Number of GET transactions for State Store within an interval time of metric |
-| `ReadsAvgTime` | Average time of GET transactions for State Store in milliseconds |
-| `WritesNumOps` | Number of PUT transactions for State Store within an interval time of metric |
-| `WritesAvgTime` | Average time of PUT transactions for State Store in milliseconds |
-| `RemovesNumOps` | Number of REMOVE transactions for State Store within an interval time of metric |
-| `RemovesAvgTime` | Average time of REMOVE transactions for State Store in milliseconds |
-| `FailuresNumOps` | Number of failed transactions for State Store within an interval time of metric |
-| `FailuresAvgTime` | Average time of failed transactions for State Store in milliseconds |
-| `Cache`*BaseRecord*`Size` | Number of store records to cache in State Store |
+| Name | Description |
+|:------------------------------------------|:-----------------------------------------------------------------------------------|
+| `ReadsNumOps` | Number of GET transactions for State Store within an interval time of metric |
+| `ReadsAvgTime` | Average time of GET transactions for State Store in milliseconds |
+| `WritesNumOps` | Number of PUT transactions for State Store within an interval time of metric |
+| `WritesAvgTime` | Average time of PUT transactions for State Store in milliseconds |
+| `RemovesNumOps` | Number of REMOVE transactions for State Store within an interval time of metric |
+| `RemovesAvgTime` | Average time of REMOVE transactions for State Store in milliseconds |
+| `FailuresNumOps` | Number of failed transactions for State Store within an interval time of metric |
+| `FailuresAvgTime` | Average time of failed transactions for State Store in milliseconds |
+| `Cache`*BaseRecord*`Size` | Number of store records to cache in State Store |
+| `Cache`*BaseRecord*`LoadNumOps` | Number of times store records are loaded in the State Store Cache from State Store |
+| `Cache`*BaseRecord*`LoadAvgTime` | Average time of loading State Store Cache from State Store in milliseconds |
yarn context
============
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
index 371b33e05e2..b5c4047acd1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.federation.metrics;
import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -54,6 +55,7 @@ public class StateStoreMetrics implements StateStoreMBean {
private MutableRate failures;
private Map<String, MutableGaugeInt> cacheSizes;
+ private final Map<String, MutableRate> cacheLoadMetrics = new HashMap<>();
protected StateStoreMetrics() {}
@@ -150,6 +152,32 @@ public class StateStoreMetrics implements StateStoreMBean {
counter.set(count);
}
+ /**
+ * Set the cache loading metrics for the state store interface.
+ *
+ * @param name Name of the record of the cache.
+ * @param value The time duration interval as the cache value.
+ */
+ public void setCacheLoading(String name, long value) {
+ String cacheLoad = "Cache" + name + "Load";
+ MutableRate cacheLoadMetric = cacheLoadMetrics.get(cacheLoad);
+ if (cacheLoadMetric == null) {
+ cacheLoadMetric = registry.newRate(cacheLoad, name, false);
+ cacheLoadMetrics.put(cacheLoad, cacheLoadMetric);
+ }
+ cacheLoadMetrics.get(cacheLoad).add(value);
+ }
+
+ /**
+ * Retrieve unmodifiable map of cache loading metrics.
+ *
+ * @return unmodifiable map of cache loading metrics.
+ */
+ @VisibleForTesting
+ public Map<String, MutableRate> getCacheLoadMetrics() {
+ return Collections.unmodifiableMap(cacheLoadMetrics);
+ }
+
@VisibleForTesting
public void reset() {
reads.resetMinMax();
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
index 6fea9b9946d..08dcc1c6e46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
@@ -113,6 +113,7 @@ public abstract class CachedRecordStore<R extends BaseRecord>
if (force || isUpdateTime()) {
List<R> newRecords = null;
long t = -1;
+ long startTime = Time.monotonicNow();
try {
QueryResult<R> result = getDriver().get(getRecordClass());
newRecords = result.getRecords();
@@ -143,6 +144,7 @@ public abstract class CachedRecordStore<R extends BaseRecord>
StateStoreMetrics metrics = getDriver().getMetrics();
if (metrics != null) {
String recordName = getRecordClass().getSimpleName();
+ metrics.setCacheLoading(recordName, Time.monotonicNow() - startTime);
metrics.setCacheSize(recordName, this.records.size());
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
index 4eb38b06b12..48d84f9326b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
@@ -48,6 +48,8 @@ import org.apache.hadoop.hdfs.server.federation.store.records.Query;
import org.apache.hadoop.hdfs.server.federation.store.records.QueryResult;
import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+
import org.junit.After;
import org.junit.AfterClass;
import org.slf4j.Logger;
@@ -76,6 +78,10 @@ public class TestStateStoreDriverBase {
return stateStore.getDriver();
}
+ protected StateStoreService getStateStoreService() {
+ return stateStore;
+ }
+
@After
public void cleanMetrics() {
if (stateStore != null) {
@@ -574,6 +580,36 @@ public class TestStateStoreDriverBase {
return getters;
}
+ public long getMountTableCacheLoadSamples(StateStoreDriver driver) throws IOException {
+ final MutableRate mountTableCache = getMountTableCache(driver);
+ return mountTableCache.lastStat().numSamples();
+ }
+
+ private static MutableRate getMountTableCache(StateStoreDriver driver) throws IOException {
+ StateStoreMetrics metrics = stateStore.getMetrics();
+ final Query<MountTable> query = new Query<>(MountTable.newInstance());
+ driver.getMultiple(MountTable.class, query);
+ final Map<String, MutableRate> cacheLoadMetrics = metrics.getCacheLoadMetrics();
+ final MutableRate mountTableCache = cacheLoadMetrics.get("CacheMountTableLoad");
+ assertNotNull("CacheMountTableLoad should be present in the state store metrics",
+ mountTableCache);
+ return mountTableCache;
+ }
+
+ public void testCacheLoadMetrics(StateStoreDriver driver, long numRefresh,
+ double expectedHigherThan) throws IOException, IllegalArgumentException {
+ final MutableRate mountTableCache = getMountTableCache(driver);
+ // CacheMountTableLoadNumOps
+ final long mountTableCacheLoadNumOps = getMountTableCacheLoadSamples(driver);
+ assertEquals("Num of samples collected should match", numRefresh, mountTableCacheLoadNumOps);
+ // CacheMountTableLoadAvgTime ms
+ final double mountTableCacheLoadAvgTimeMs = mountTableCache.lastStat().mean();
+ assertTrue(
+ "Mean time duration for cache load is expected to be higher than " + expectedHigherThan
+ + " ms." + " Actual value: " + mountTableCacheLoadAvgTimeMs,
+ mountTableCacheLoadAvgTimeMs > expectedHigherThan);
+ }
+
/**
* Get the type of field.
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
index a8a9020744c..b01500b2ea1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFile.java
@@ -73,4 +73,16 @@ public class TestStateStoreFile extends TestStateStoreDriverBase {
throws IllegalArgumentException, IllegalAccessException, IOException {
testMetrics(getStateStoreDriver());
}
+
+ @Test
+ public void testCacheLoadMetrics() throws IOException {
+ // inject value of CacheMountTableLoad as -1 initially, if tests get CacheMountTableLoadAvgTime
+ // value as -1 ms, that would mean no other sample with value >= 0 would have been received and
+ // hence this would be failure to assert that mount table avg load time is higher than -1
+ getStateStoreService().getMetrics().setCacheLoading("MountTable", -1);
+ long curMountTableLoadNum = getMountTableCacheLoadSamples(getStateStoreDriver());
+ getStateStoreService().refreshCaches(true);
+ testCacheLoadMetrics(getStateStoreDriver(), curMountTableLoadNum + 1, -1);
+ }
+
}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
index dbd4b9bdae2..8c06e6b8ed1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
@@ -115,4 +115,16 @@ public class TestStateStoreFileSystem extends TestStateStoreDriverBase {
testInsertWithErrorDuringWrite(driver, MembershipState.class);
}
+
+ @Test
+ public void testCacheLoadMetrics() throws IOException {
+ // inject value of CacheMountTableLoad as -1 initially, if tests get CacheMountTableLoadAvgTime
+ // value as -1 ms, that would mean no other sample with value >= 0 would have been received and
+ // hence this would be failure to assert that mount table avg load time is higher than -1
+ getStateStoreService().getMetrics().setCacheLoading("MountTable", -1);
+ long curMountTableLoadNum = getMountTableCacheLoadSamples(getStateStoreDriver());
+ getStateStoreService().refreshCaches(true);
+ getStateStoreService().refreshCaches(true);
+ testCacheLoadMetrics(getStateStoreDriver(), curMountTableLoadNum + 2, -1);
+ }
}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
index 3ad106697ac..f94e415b4d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreZK.java
@@ -206,4 +206,18 @@ public class TestStateStoreZK extends TestStateStoreDriverBase {
stateStoreDriver.setEnableConcurrent(true);
testFetchErrors(stateStoreDriver);
}
+
+ @Test
+ public void testCacheLoadMetrics() throws IOException {
+ // inject value of CacheMountTableLoad as -1 initially, if tests get CacheMountTableLoadAvgTime
+ // value as -1 ms, that would mean no other sample with value >= 0 would have been received and
+ // hence this would be failure to assert that mount table avg load time is higher than -1
+ getStateStoreService().getMetrics().setCacheLoading("MountTable", -1);
+ long curMountTableLoadNum = getMountTableCacheLoadSamples(getStateStoreDriver());
+ getStateStoreService().refreshCaches(true);
+ getStateStoreService().refreshCaches(true);
+ getStateStoreService().refreshCaches(true);
+ testCacheLoadMetrics(getStateStoreDriver(), curMountTableLoadNum + 3, -1);
+ }
+
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org