You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by we...@apache.org on 2026/04/30 15:48:04 UTC
(ozone) branch master updated: HDDS-13133. Display Ratis state machine event timeline in OM web UI (#10034)
This is an automated email from the ASF dual-hosted git repository.
jojochuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 99b53f44436 HDDS-13133. Display Ratis state machine event timeline in OM web UI (#10034)
99b53f44436 is described below
commit 99b53f44436638db53bef242f65e6f8e385be99d
Author: Wei-Chiu Chuang <we...@apache.org>
AuthorDate: Thu Apr 30 08:47:57 2026 -0700
HDDS-13133. Display Ratis state machine event timeline in OM web UI (#10034)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 ++
.../common/src/main/resources/ozone-default.xml | 12 +++++
.../resources/webapps/static/templates/menu.html | 1 +
.../container/placement/metrics/SCMMetrics.java | 36 ++++++++++++++-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 31 +++++++++++--
.../hdds/scm/server/SCMBlockProtocolServer.java | 3 +-
.../hdds/scm/server/StorageContainerManager.java | 16 +++----
.../main/resources/webapps/scm/ratis-events.html | 32 +++++++++++++
.../src/main/resources/webapps/scm/scm.js | 26 +++++++++++
.../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 52 ++++++++++++++++++++++
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 4 ++
.../hdds/scm/TestSCMDbCheckpointServlet.java | 2 +-
.../ozone/om/service/TestRangerBGSyncService.java | 2 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 33 ++++++++++++--
.../org/apache/hadoop/ozone/om/OzoneManager.java | 2 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 49 +++++++++++++++++++-
.../resources/webapps/ozoneManager/ozoneManager.js | 21 +++++++++
.../webapps/ozoneManager/ratis-events.html | 32 +++++++++++++
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 25 +++++++++++
.../request/TestOMClientRequestWithUserInfo.java | 2 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 2 +-
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 2 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 4 +-
.../s3/multipart/TestS3MultipartRequest.java | 2 +-
.../s3/security/TestS3GetSecretRequest.java | 2 +-
.../s3/tenant/TestOMTenantCreateRequest.java | 2 +-
.../s3/tenant/TestOMTenantDeleteRequest.java | 2 +-
.../om/request/volume/TestOMVolumeRequest.java | 2 +-
.../ozone/om/snapshot/TestSnapshotCache.java | 8 ++--
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 4 +-
.../snapshot/TestSnapshotRequestAndResponse.java | 4 +-
34 files changed, 383 insertions(+), 42 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index a90bacc9fee..e0b28548e8e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -629,6 +629,10 @@ public final class ScmConfigKeys {
= "ozone.scm.ha.raft.server.rpc.first-election.timeout";
public static final String HDDS_SCM_HTTP_AUTH_TYPE = "hdds.scm.http.auth.type";
+ public static final String OZONE_SCM_RATIS_EVENTS_MAX_LIMIT =
+ "ozone.scm.ratis.events.max.limit";
+ public static final int OZONE_SCM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100;
+
/**
* Never constructed.
*/
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index eeeb3f03051..aeda046dcd7 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -5016,4 +5016,16 @@
<value>5m</value>
<description>Interval for cleaning up orphan snapshot local data versions corresponding to snapshots</description>
</property>
+ <property>
+ <name>ozone.scm.ratis.events.max.limit</name>
+ <value>100</value>
+ <tag>OZONE, RATIS, SCM</tag>
+ <description>The maximum number of events that can be pending in SCM Ratis.</description>
+ </property>
+ <property>
+ <name>ozone.om.ratis.events.max.limit</name>
+ <value>100</value>
+ <tag>OZONE, RATIS, OM</tag>
+ <description>The maximum number of events that can be pending in OM Ratis.</description>
+ </property>
</configuration>
diff --git a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html
index d0a6907e586..b488a6982e7 100644
--- a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html
+++ b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html
@@ -34,6 +34,7 @@
</ul>
</li>
<li><a href="#!/config">Configuration</a></li>
+ <li><a href="#!/ratis_events">Ratis event timeline</a></li>
<li ng-show="$ctrl.docs"><a href="/docs">Documentation</a></li>
<li>
<a class="dropdown-toggle"
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java
index feb4129952d..d5dab7800c4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java
@@ -17,6 +17,9 @@
package org.apache.hadoop.hdds.scm.container.placement.metrics;
+import java.util.LinkedList;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.utils.DBCheckpointMetrics;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
@@ -24,6 +27,7 @@
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
+import org.apache.hadoop.util.Time;
/**
* This class is for maintaining StorageContainerManager statistics.
@@ -33,6 +37,9 @@ public class SCMMetrics {
public static final String SOURCE_NAME =
SCMMetrics.class.getSimpleName();
+ private final LinkedList<String> ratisEvents = new LinkedList<>();
+ private final int maxRatisEvents;
+
/**
* Container stat metrics, the meaning of following metrics
* can be found in {@link ContainerStat}.
@@ -59,14 +66,23 @@ public DBCheckpointMetrics getDBCheckpointMetrics() {
return dbCheckpointMetrics;
}
- public SCMMetrics() {
+ public SCMMetrics(int maxRatisEvents) {
dbCheckpointMetrics = DBCheckpointMetrics.create("SCM Metrics");
+ this.maxRatisEvents = maxRatisEvents;
}
public static SCMMetrics create() {
+ return create(null);
+ }
+
+ public static SCMMetrics create(ConfigurationSource conf) {
MetricsSystem ms = DefaultMetricsSystem.instance();
+ int maxRatisEvents = conf == null
+ ? ScmConfigKeys.OZONE_SCM_RATIS_EVENTS_MAX_LIMIT_DEFAULT
+ : conf.getInt(ScmConfigKeys.OZONE_SCM_RATIS_EVENTS_MAX_LIMIT,
+ ScmConfigKeys.OZONE_SCM_RATIS_EVENTS_MAX_LIMIT_DEFAULT);
return ms.register(SOURCE_NAME, "Storage Container Manager Metrics",
- new SCMMetrics());
+ new SCMMetrics(maxRatisEvents));
}
public void setLastContainerReportSize(long size) {
@@ -155,6 +171,22 @@ public void decrContainerStat(ContainerStat deltaStat) {
this.containerReportWriteCount.incr(-1 * deltaStat.getWriteCount().get());
}
+ public void addRatisEvent(String event) {
+ synchronized (ratisEvents) {
+ if (ratisEvents.size() >= maxRatisEvents) {
+ ratisEvents.removeFirst();
+ }
+ ratisEvents.add(Time.formatTime(Time.now()) + "|" + event);
+ }
+ }
+
+ @Metric("Ratis state machine events")
+ public String getRatisEvents() {
+ synchronized (ratisEvents) {
+ return String.join("\n", ratisEvents);
+ }
+ }
+
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
index d0f52479f40..f43ce2c3dc0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
@@ -39,6 +39,7 @@
import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
import org.apache.hadoop.hdds.scm.block.DeletedBlockLog;
import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.ha.invoker.ScmInvoker;
@@ -80,6 +81,7 @@ public class SCMStateMachine extends BaseStateMachine {
private Map<RequestType, Object> handlers;
private Map<RequestType, ScmInvoker<?>> invokers;
private SCMHADBTransactionBuffer transactionBuffer;
+ private final SCMMetrics metrics;
private final SimpleStateMachineStorage storage =
new SimpleStateMachineStorage();
private final boolean isInitialized;
@@ -97,6 +99,7 @@ public SCMStateMachine(final StorageContainerManager scm,
this.handlers = new EnumMap<>(RequestType.class);
this.invokers = new EnumMap<>(RequestType.class);
this.transactionBuffer = buffer;
+ this.metrics = scm.getMetrics();
TransactionInfo latestTrxInfo = this.transactionBuffer.getLatestTrxInfo();
if (!latestTrxInfo.isDefault()) {
updateLastAppliedTermIndex(latestTrxInfo.getTerm(),
@@ -114,12 +117,19 @@ public SCMStateMachine(final StorageContainerManager scm,
public SCMStateMachine() {
isInitialized = false;
+ this.metrics = null;
}
public void registerHandler(RequestType type, Object handler) {
handlers.put(type, handler);
}
+ private void addRatisEvent(String message) {
+ if (metrics != null) {
+ metrics.addRatisEvent(message);
+ }
+ }
+
public void registerInvoker(RequestType type,
ScmInvoker<?> invoker) {
invokers.put(type, invoker);
@@ -224,7 +234,9 @@ public void notifyNotLeader(Collection<TransactionContext> pendingEntries) {
if (!isInitialized) {
return;
}
- LOG.info("current leader SCM steps down.");
+ String message = "SCM " + scm.getScmId() + " steps down from being leader.";
+ LOG.info(message);
+ addRatisEvent(message);
scm.getScmContext().updateLeaderAndTerm(false, 0);
scm.getSCMServiceManager().notifyStatusChanged();
@@ -255,6 +267,8 @@ public CompletableFuture<TermIndex> notifyInstallSnapshotFromLeader(
final String leaderNodeId = leaderDetails.get().getNodeId();
LOG.info("Received install snapshot notification from SCM leader: {} with "
+ "term index: {}", leaderAddress, firstTermIndexInLog);
+ addRatisEvent("Installing snapshot from SCM leader " + leaderNodeId +
+ ", term index: " + firstTermIndexInLog);
CompletableFuture<TermIndex> future = CompletableFuture.supplyAsync(
() -> {
@@ -308,11 +322,17 @@ public void notifyLeaderChanged(RaftGroupMemberId groupMemberId,
}
if (!groupMemberId.getPeerId().equals(newLeaderId)) {
- LOG.info("leader changed, yet current SCM is still follower.");
+ String message = "Leader changed to " + newLeaderId +
+ ", current SCM " + scm.getScmId() + " is still follower.";
+ LOG.info(message);
+ addRatisEvent(message);
return;
}
- LOG.info("current SCM becomes leader of term {}.", currentLeaderTerm);
+ String message = "current SCM " + scm.getScmId() +
+ " becomes leader of term " + currentLeaderTerm;
+ LOG.info(message);
+ addRatisEvent(message);
scm.getScmContext().updateLeaderAndTerm(true,
currentLeaderTerm.get());
@@ -406,11 +426,15 @@ public void notifyLeaderReady() {
scm.getScmContext().setLeaderReady();
scm.getSCMServiceManager().notifyStatusChanged();
scm.getFinalizationManager().onLeaderReady();
+ addRatisEvent("SCM " + scm.getScmId() +
+ " is ready to serve requests as the leader");
}
@Override
public void notifyConfigurationChanged(long term, long index,
RaftProtos.RaftConfigurationProto newRaftConfiguration) {
+ addRatisEvent("Configuration changed at term index (" + term + ", " + index +
+ ") to " + newRaftConfiguration.toString());
}
@Override
@@ -443,6 +467,7 @@ public void reinitialize() throws IOException {
}
LOG.info("{}: SCMStateMachine is reinitializing. newTermIndex = {}", getId(), termIndex);
+ addRatisEvent("reinitialize: " + termIndex);
// re-initialize the DBTransactionBuffer and update the lastAppliedIndex.
try {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
index 1fc7c2d96b2..e563c9f08ff 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
@@ -25,7 +25,6 @@
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.IO_EXCEPTION;
import static org.apache.hadoop.hdds.scm.net.NetConstants.NODE_COST_DEFAULT;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
-import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.getPerfMetrics;
import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName;
import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
@@ -108,7 +107,7 @@ public class SCMBlockProtocolServer implements
public SCMBlockProtocolServer(OzoneConfiguration conf,
StorageContainerManager scm) throws IOException {
this.scm = scm;
- this.perfMetrics = getPerfMetrics();
+ this.perfMetrics = scm.getPerfMetrics();
final int handlerCount = conf.getInt(OZONE_SCM_BLOCK_HANDLER_COUNT_KEY,
OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT,
LOG::info);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 696816b85a2..69f7973ff1b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -226,8 +226,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
/**
* SCM metrics.
*/
- private static SCMMetrics metrics;
- private static SCMPerformanceMetrics perfMetrics;
+ private SCMMetrics metrics;
+ private SCMPerformanceMetrics perfMetrics;
private SCMHAMetrics scmHAMetrics;
private final NettyMetrics nettyMetrics;
@@ -1427,28 +1427,28 @@ private static InetSocketAddress getScmAddress(SCMHANodeDetails haDetails,
/**
* Initialize SCM metrics.
*/
- public static void initMetrics() {
- metrics = SCMMetrics.create();
+ public void initMetrics() {
+ metrics = SCMMetrics.create(configuration);
}
/**
* Return SCM metrics instance.
*/
- public static SCMMetrics getMetrics() {
- return metrics == null ? SCMMetrics.create() : metrics;
+ public SCMMetrics getMetrics() {
+ return metrics;
}
/**
* Initialize SCMPerformance metrics.
*/
- public static void initPerfMetrics() {
+ public void initPerfMetrics() {
perfMetrics = SCMPerformanceMetrics.create();
}
/**
* Return SCMPerformance metrics instance.
*/
- public static SCMPerformanceMetrics getPerfMetrics() {
+ public SCMPerformanceMetrics getPerfMetrics() {
return perfMetrics == null ? SCMPerformanceMetrics.create() : perfMetrics;
}
diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/ratis-events.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/ratis-events.html
new file mode 100644
index 00000000000..9eb6008d00f
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/ratis-events.html
@@ -0,0 +1,32 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<h1>Ratis event timeline</h1>
+
+<table class="table table-striped">
+ <thead>
+ <tr>
+ <th class="col-md-3">Timestamp</th>
+ <th class="col-md-9">Event description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="event in $ctrl.events track by $index">
+ <td>{{event.timestamp}}</td>
+ <td>{{event.description}}</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
index eca79852e43..1e44826f980 100644
--- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
+++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
@@ -19,6 +19,32 @@
"use strict";
angular.module('scm', ['ozone', 'nvd3']);
+ angular.module('scm').config(function ($routeProvider) {
+ $routeProvider
+ .when("/ratis_events", {
+ template: "<ratis-events></ratis-events>"
+ });
+ });
+
+ angular.module('scm').component('ratisEvents', {
+ templateUrl: 'ratis-events.html',
+ controller: function ($http) {
+ var ctrl = this;
+ $http.get("jmx?qry=Hadoop:service=StorageContainerManager,name=SCMMetrics")
+ .then(function (result) {
+ var metrics = result.data.beans[0];
+ var rawEvents = metrics['tag.RatisEvents'] ? metrics['tag.RatisEvents'].split('\n') : [];
+ ctrl.events = rawEvents.map(function(e) {
+ var parts = e.split('|');
+ return {
+ timestamp: parts[0],
+ description: parts[1]
+ };
+ });
+ });
+ }
+ });
+
angular.module('scm').component('scmOverview', {
templateUrl: 'scm-overview.html',
require: {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java
new file mode 100644
index 00000000000..828606f2d42
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.ha;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.utils.TransactionInfo;
+import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.server.protocol.TermIndex;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test SCMStateMachine events recording.
+ */
+public class TestSCMStateMachine {
+
+ @Test
+ public void testRatisEventsRecording() throws Exception {
+ StorageContainerManager scm = mock(StorageContainerManager.class);
+ SCMMetrics metrics = SCMMetrics.create();
+ when(scm.getMetrics()).thenReturn(metrics);
+
+ SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class);
+ when(buffer.getLatestTrxInfo()).thenReturn(TransactionInfo.valueOf(TermIndex.valueOf(0, 0)));
+
+ SCMStateMachine stateMachine = new SCMStateMachine(scm, buffer);
+
+ stateMachine.notifyConfigurationChanged(1, 1, RaftProtos.RaftConfigurationProto.getDefaultInstance());
+ assertTrue(metrics.getRatisEvents().contains("Configuration changed at term index"));
+
+ metrics.unRegister();
+ }
+}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index e24a9036b2d..b9ccfedb40d 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -694,6 +694,10 @@ public final class OMConfigKeys {
"ozone.om.snapshot.local.data.manager.service.interval";
public static final String OZONE_OM_SNAPSHOT_LOCAL_DATA_MANAGER_SERVICE_INTERVAL_DEFAULT = "5m";
+ public static final String OZONE_OM_RATIS_EVENTS_MAX_LIMIT =
+ "ozone.om.ratis.events.max.limit";
+ public static final int OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100;
+
/**
* Never constructed.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java
index cb8045e0e6e..7054e350682 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java
@@ -87,7 +87,7 @@ public void init() throws Exception {
.build();
cluster.waitForClusterToBeReady();
StorageContainerManager scm = cluster.getStorageContainerManager();
- scmMetrics = StorageContainerManager.getMetrics();
+ scmMetrics = scm.getMetrics();
requestMock = mock(HttpServletRequest.class);
when(requestMock.getParameter(OZONE_DB_CHECKPOINT_REQUEST_FLUSH))
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRangerBGSyncService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRangerBGSyncService.java
index 9f5690e11e4..a01bcf37048 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRangerBGSyncService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestRangerBGSyncService.java
@@ -177,7 +177,7 @@ public void setUp() throws IOException {
Path metaDirPath = Paths.get(path, "om-meta");
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
- omMetrics = OMMetrics.create();
+ omMetrics = OMMetrics.create(conf);
conf.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.resolve("om").toAbsolutePath().toString());
// No need to conf.set(OzoneConfigKeys.OZONE_ADMINISTRATORS, ...) here
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
index 9d2cab2fc50..5a70483b2bc 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.ozone.om;
import com.google.common.annotations.VisibleForTesting;
+import java.util.LinkedList;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.DBCheckpointMetrics;
import org.apache.hadoop.metrics2.MetricsSystem;
@@ -28,6 +30,7 @@
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.ozone.om.snapshot.OMSnapshotDirectoryMetrics;
+import org.apache.hadoop.util.Time;
/**
* This class is for maintaining Ozone Manager statistics.
@@ -38,6 +41,9 @@ public class OMMetrics implements OmMetadataReaderMetrics {
private static final String SOURCE_NAME =
OMMetrics.class.getSimpleName();
+ private final LinkedList<String> ratisEvents = new LinkedList<>();
+ private final int maxRatisEvents;
+
// OM request type op metrics
private @Metric MutableCounterLong numVolumeOps;
private @Metric MutableCounterLong numBucketOps;
@@ -256,15 +262,20 @@ public class OMMetrics implements OmMetadataReaderMetrics {
private final DBCheckpointMetrics dbCheckpointMetrics;
private OMSnapshotDirectoryMetrics snapshotDirectoryMetrics;
- public OMMetrics() {
+ public OMMetrics(int maxRatisEvents) {
dbCheckpointMetrics = DBCheckpointMetrics.create("OM Metrics");
+ this.maxRatisEvents = maxRatisEvents;
}
- public static OMMetrics create() {
+ public static OMMetrics create(ConfigurationSource conf) {
MetricsSystem ms = DefaultMetricsSystem.instance();
+ int maxRatisEvents = conf == null
+ ? OMConfigKeys.OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT
+ : conf.getInt(OMConfigKeys.OZONE_OM_RATIS_EVENTS_MAX_LIMIT,
+ OMConfigKeys.OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT);
return ms.register(SOURCE_NAME,
"Ozone Manager Metrics",
- new OMMetrics());
+ new OMMetrics(maxRatisEvents));
}
public DBCheckpointMetrics getDBCheckpointMetrics() {
@@ -1570,6 +1581,22 @@ public void incNumRecoverLeaseFails() {
numRecoverLeaseFails.incr();
}
+ public void addRatisEvent(String event) {
+ synchronized (ratisEvents) {
+ if (ratisEvents.size() >= maxRatisEvents) {
+ ratisEvents.removeFirst();
+ }
+ ratisEvents.add(Time.formatTime(Time.now()) + "|" + event);
+ }
+ }
+
+ @Metric("Ratis state machine events")
+ public String getRatisEvents() {
+ synchronized (ratisEvents) {
+ return String.join("\n", ratisEvents);
+ }
+ }
+
public void unRegister() {
if (dbCheckpointMetrics != null) {
dbCheckpointMetrics.unRegister();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index cabf24066eb..bb25b48ec40 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -695,7 +695,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
this.isS3MultiTenancyEnabled =
OMMultiTenantManager.checkAndEnableMultiTenancy(this, conf);
- metrics = OMMetrics.create();
+ metrics = OMMetrics.create(conf);
omSnapshotIntMetrics = OmSnapshotInternalMetrics.create();
perfMetrics = OMPerformanceMetrics.register();
omDeletionMetrics = DeletingServiceMetrics.create();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
index 09a530ab6cc..2abaf9ae571 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
@@ -26,6 +26,7 @@
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -41,6 +42,7 @@
import org.apache.hadoop.ozone.audit.AuditLoggerType;
import org.apache.hadoop.ozone.audit.OMSystemAction;
import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.OzoneManagerPrepareState;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -170,6 +172,10 @@ public synchronized void reinitialize() throws IOException {
final TermIndex lastApplied = getLastAppliedTermIndex();
unpause(lastApplied.getIndex(), lastApplied.getTerm());
LOG.info("{}: reinitialize {} with {}", getId(), getGroupId(), lastApplied);
+ OMMetrics metrics = ozoneManager.getMetrics();
+ if (metrics != null) {
+ metrics.addRatisEvent("reinitialize: " + lastApplied);
+ }
}
}
@@ -183,6 +189,19 @@ public SnapshotInfo getLatestSnapshot() {
@Override
public void notifyLeaderReady() {
ozoneManager.getOmSnapshotManager().resetInFlightSnapshotCount();
+ OMMetrics metrics = ozoneManager.getMetrics();
+ if (metrics != null) {
+ metrics.addRatisEvent("Ready to serve requests as the leader");
+ }
+ }
+
+ @Override
+ public void notifyNotLeader(Collection<TransactionContext> pendingEntries) {
+ LOG.info("current leader OM steps down.");
+ OMMetrics metrics = ozoneManager.getMetrics();
+ if (metrics != null) {
+ metrics.addRatisEvent("current leader OM steps down.");
+ }
}
@Override
@@ -208,6 +227,10 @@ public void notifyLeaderChanged(RaftGroupMemberId groupMemberId,
AUDIT.logWriteSuccess(ozoneManager.buildAuditMessageForSuccess(OMSystemAction.LEADER_CHANGE, auditParams));
LOG.info("{}: leader changed to {}", groupMemberId, newLeaderId);
+ OMMetrics metrics = ozoneManager.getMetrics();
+ if (metrics != null) {
+ metrics.addRatisEvent("Leader changed to " + newLeaderId);
+ }
}
/** Notified by Ratis for non-StateMachine term-index update. */
@@ -282,10 +305,22 @@ public void notifyConfigurationChanged(long term, long index,
for (RaftProtos.RaftPeerProto raftPeerProto : newPeers) {
newPeerIds.add(RaftPeerId.valueOf(raftPeerProto.getId()).toString());
}
- for (RaftProtos.RaftPeerProto raftPeerProto : newListeners) {
- newPeerIds.add(RaftPeerId.valueOf(raftPeerProto.getId()).toString());
+ List<String> newListenersIds = new ArrayList<>();
+ for (RaftProtos.RaftPeerProto raftListenerProto : newListeners) {
+ newListenersIds.add(RaftPeerId.valueOf(raftListenerProto.getId()).toString());
+ }
+
+ OMMetrics metrics = ozoneManager.getMetrics();
+ if (metrics != null) {
+ metrics.addRatisEvent(
+ "New peers " + newPeerIds +
+ (newListenersIds.isEmpty() ? "" : ", new listeners " + newListenersIds) +
+ " added at term index (" +
+ term + ", " + index + ")");
}
+
// Check and update the peer list in OzoneManager
+ newPeerIds.addAll(newListenersIds);
ozoneManager.updatePeerList(newPeerIds);
}
@@ -301,6 +336,11 @@ public void notifySnapshotInstalled(RaftProtos.InstallSnapshotResult result,
long snapshotIndex, RaftPeer peer) {
LOG.info("Receive notifySnapshotInstalled event {} for the peer: {}" +
" snapshotIndex: {}.", result, peer.getId(), snapshotIndex);
+ OMMetrics metrics = ozoneManager.getMetrics();
+ if (metrics != null) {
+ metrics.addRatisEvent("Install snapshot " +
+ result + ", snapshotIndex=" + snapshotIndex + ", peer=" + peer.getId());
+ }
switch (result) {
case SUCCESS:
case SNAPSHOT_UNAVAILABLE:
@@ -581,6 +621,11 @@ public CompletableFuture<TermIndex> notifyInstallSnapshotFromLeader(
.getLeaderInfo().getId().getId()).toString();
LOG.info("Received install snapshot notification from OM leader: {} with " +
"term index: {}", leaderNodeId, firstTermIndexInLog);
+ OMMetrics metrics = ozoneManager.getMetrics();
+ if (metrics != null) {
+ metrics.addRatisEvent("Installing snapshot from " +
+ "OM leader " + leaderNodeId + ", term index: " + firstTermIndexInLog);
+ }
return CompletableFuture.supplyAsync(
() -> {
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
index 8ad270e8304..fdbc300b84d 100644
--- a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
@@ -30,6 +30,9 @@
})
.when("/snapshots", {
template: "<om-snapshots></om-snapshots>"
+ })
+ .when("/ratis_events", {
+ template: "<ratis-events></ratis-events>"
});
});
angular.module('ozoneManager').component('omSnapshots', {
@@ -163,6 +166,24 @@
}
}
});
+ angular.module('ozoneManager').component('ratisEvents', {
+ templateUrl: 'ratis-events.html',
+ controller: function ($http) {
+ var ctrl = this;
+ $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics")
+ .then(function (result) {
+ var metrics = result.data.beans[0];
+ var rawEvents = metrics['tag.RatisEvents'] ? metrics['tag.RatisEvents'].split('\n') : [];
+ ctrl.events = rawEvents.map(function(e) {
+ var parts = e.split('|');
+ return {
+ timestamp: parts[0],
+ description: parts[1]
+ };
+ });
+ });
+ }
+ });
angular.module('ozoneManager').component('omMetrics', {
templateUrl: 'om-metrics.html',
controller: function ($http) {
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ratis-events.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ratis-events.html
new file mode 100644
index 00000000000..9eb6008d00f
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ratis-events.html
@@ -0,0 +1,32 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<h1>Ratis event timeline</h1>
+
+<table class="table table-striped">
+ <thead>
+ <tr>
+ <th class="col-md-3">Timestamp</th>
+ <th class="col-md-9">Event description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="event in $ctrl.events track by $index">
+ <td>{{event.timestamp}}</td>
+ <td>{{event.description}}</td>
+ </tr>
+ </tbody>
+</table>
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java
index 81a0691a749..c0ad927bc05 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java
@@ -106,8 +106,8 @@ class TestOzoneManagerDoubleBuffer {
@BeforeEach
public void setup() throws IOException {
- OMMetrics omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ OMMetrics omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
tempDir.getAbsolutePath());
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java
index 4ba3c6b5d1b..8b3027d12cd 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java
@@ -91,8 +91,8 @@ public class TestOzoneManagerDoubleBufferWithOMResponse {
@BeforeEach
public void setup() throws IOException {
ozoneManager = mock(OzoneManager.class, withSettings().stubOnly());
- OMMetrics omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ OMMetrics omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
index 36d7a80aeea..111779b9573 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java
@@ -54,6 +54,7 @@
import org.apache.hadoop.ozone.audit.OMSystemAction;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OmConfig;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
@@ -1037,4 +1038,28 @@ private RaftClientRequest buildClientRequest(
.setType(RaftClientRequest.writeRequestType())
.build();
}
+
+ @Test
+ public void testRatisEventsRecording() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ OMMetrics metrics = OMMetrics.create(conf);
+ when(om.getMetrics()).thenReturn(metrics);
+ when(om.getOmSnapshotManager()).thenReturn(mock(OmSnapshotManager.class));
+ when(om.getConfiguration()).thenReturn(conf);
+ AuditMessage auditMessage = mock(AuditMessage.class);
+ when(auditMessage.getOp()).thenReturn("LEADER_CHANGE");
+ when(om.buildAuditMessageForSuccess(any(), any())).thenReturn(auditMessage);
+ sm.notifyLeaderReady();
+ assertTrue(metrics.getRatisEvents().contains("Ready to serve requests as the leader"));
+
+ RaftGroupMemberId groupMemberId = mock(RaftGroupMemberId.class);
+ when(groupMemberId.getPeerId()).thenReturn(RaftPeerId.valueOf("peer1"));
+ sm.notifyLeaderChanged(groupMemberId, RaftPeerId.valueOf("peer1"));
+ assertTrue(metrics.getRatisEvents().contains("Leader changed to peer1"));
+
+ sm.notifyConfigurationChanged(1, 1, RaftProtos.RaftConfigurationProto.getDefaultInstance());
+ assertTrue(metrics.getRatisEvents().contains("New peers [] added at term index "));
+
+ metrics.unRegister();
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java
index 145ffe9854f..38036912b96 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java
@@ -70,8 +70,8 @@ public class TestOMClientRequestWithUserInfo {
@BeforeEach
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- OMMetrics omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ OMMetrics omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
OMMetadataManager omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
index 018e60633a5..40f54be4cc0 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java
@@ -59,8 +59,8 @@ public class TestBucketRequest {
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
index 1b096070ea3..7adf92518d1 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java
@@ -92,8 +92,8 @@ public class TestOMDirectoryCreateRequest {
@BeforeEach
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
index 164ccf661cc..8f5b6c807ca 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java
@@ -92,8 +92,8 @@ public class TestOMDirectoryCreateRequestWithFSO {
@BeforeEach
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
OMRequestTestUtils.configureFSOptimizedPaths(ozoneConfiguration, true);
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index b84294370c5..405cc706ef9 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@ -136,10 +136,10 @@ public class TestOMKeyRequest {
@BeforeEach
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
+ OzoneConfiguration ozoneConfiguration = getOzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
perfMetrics = OMPerformanceMetrics.register();
delMetrics = DeletingServiceMetrics.create();
- OzoneConfiguration ozoneConfiguration = getOzoneConfiguration();
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
index 38749f6812a..15ed924cc40 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java
@@ -77,8 +77,8 @@ public class TestS3MultipartRequest {
@BeforeEach
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3GetSecretRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3GetSecretRequest.java
index 78ccef961f3..0067b2b3804 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3GetSecretRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3GetSecretRequest.java
@@ -133,8 +133,8 @@ public void setUp() throws Exception {
when(call.getRemoteUser()).thenReturn(ugiAlice);
Server.getCurCall().set(call);
- omMetrics = OMMetrics.create();
OzoneConfiguration conf = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(conf);
conf.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
// No need to conf.set(OzoneConfigKeys.OZONE_ADMINISTRATORS, ...) here
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantCreateRequest.java
index 36529d01064..4f1ff1659c1 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantCreateRequest.java
@@ -72,8 +72,8 @@ public class TestOMTenantCreateRequest {
@BeforeEach
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantDeleteRequest.java
index 6de4f1d8446..8accc44fd9d 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/tenant/TestOMTenantDeleteRequest.java
@@ -69,8 +69,8 @@ public class TestOMTenantDeleteRequest {
@BeforeEach
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeRequest.java
index 36c6034207c..71cb1b16627 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeRequest.java
@@ -61,8 +61,8 @@ public class TestOMVolumeRequest {
@BeforeEach
public void setup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java
index ebc78b26c1e..96fdb35d467 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java
@@ -82,6 +82,7 @@ class TestSnapshotCache {
private static IOzoneManagerLock lock;
private SnapshotCache snapshotCache;
+ private OzoneConfiguration conf;
private OMMetrics omMetrics;
@BeforeAll
@@ -94,8 +95,9 @@ static void beforeAll() throws Exception {
@BeforeEach
void setUp() throws Exception {
+ conf = new OzoneConfiguration();
// Reset cache for each test case
- omMetrics = OMMetrics.create();
+ omMetrics = OMMetrics.create(conf);
// Create a difference mock OmSnapshot instance each time load() is called
doAnswer((Answer<OmSnapshot>) invocation -> {
final OmSnapshot omSnapshot = mock(OmSnapshot.class);
@@ -197,7 +199,7 @@ public void testLockHoldsWriteLock(int numberOfLocks) {
@Test
public void testLockSupplierReturnsLockWithAnotherLockReleased() {
- IOzoneManagerLock ozoneManagerLock = new OzoneManagerLock(new OzoneConfiguration());
+ IOzoneManagerLock ozoneManagerLock = new OzoneManagerLock(conf);
snapshotCache = new SnapshotCache(cacheLoader, CACHE_SIZE_LIMIT, omMetrics, 50, true, ozoneManagerLock);
try (UncheckedAutoCloseableSupplier<OMLockDetails> lockDetails = snapshotCache.lock()) {
ozoneManagerLock.acquireWriteLock(VOLUME_LOCK, "vol1");
@@ -460,7 +462,7 @@ void testEviction3WithClose() throws IOException, InterruptedException, TimeoutE
@Test
@DisplayName("Snapshot operations not blocked during compaction")
void testSnapshotOperationsNotBlockedDuringCompaction() throws IOException, InterruptedException, TimeoutException {
- omMetrics = OMMetrics.create();
+ omMetrics = OMMetrics.create(conf);
snapshotCache = new SnapshotCache(cacheLoader, 1, omMetrics, 50, true,
lock);
final UUID dbKey1 = UUID.randomUUID();
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
index f32e3e9d4bb..ddebf54d600 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
@@ -194,7 +194,6 @@ public class TestSnapshotDiffManager {
private final List<String> snapshotNames = new ArrayList<>();
private final List<SnapshotInfo> snapshotInfoList = new ArrayList<>();
private final List<SnapshotDiffJob> snapDiffJobs = new ArrayList<>();
- private final OMMetrics omMetrics = OMMetrics.create();
@TempDir
private File dbDir;
@Mock
@@ -235,6 +234,9 @@ public void init() throws RocksDBException, IOException, ExecutionException {
ExitUtils.disableSystemExit();
ExitUtil.disableSystemExit();
+ OzoneConfiguration conf = new OzoneConfiguration();
+ OMMetrics omMetrics = OMMetrics.create(conf);
+
dbOptions = new ManagedDBOptions();
dbOptions.setCreateIfMissing(true);
columnFamilyOptions = new ManagedColumnFamilyOptions();
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java
index 9c6f033b907..e3776a5b837 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java
@@ -142,9 +142,9 @@ protected TestSnapshotRequestAndResponse(boolean isAdmin) {
@BeforeEach
public void baseSetup() throws Exception {
ozoneManager = mock(OzoneManager.class);
- omMetrics = OMMetrics.create();
- omSnapshotIntMetrics = OmSnapshotInternalMetrics.create();
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
+ omSnapshotIntMetrics = OmSnapshotInternalMetrics.create();
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
testDir.getAbsolutePath());
ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org