You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2022/03/02 17:23:46 UTC
[ozone] branch master updated: HDDS-6333. Add a metric to record sequence number lag between Recon and OM (#3100)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ba31f8c HDDS-6333. Add a metric to record sequence number lag between Recon and OM (#3100)
ba31f8c is described below
commit ba31f8ce76b5a429e9ac572c644e177f3d85d8cc
Author: Symious <yi...@foxmail.com>
AuthorDate: Thu Mar 3 01:23:29 2022 +0800
HDDS-6333. Add a metric to record sequence number lag between Recon and OM (#3100)
---
.../org/apache/hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 +++++++++
.../java/org/apache/hadoop/hdds/utils/db/RDBStore.java | 1 +
.../java/org/apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 ++++++++++
.../OzoneManagerProtocolClientSideTranslatorPB.java | 2 ++
.../hadoop/ozone/recon/TestReconWithOzoneManager.java | 5 +++++
.../interface-client/src/main/proto/OmClientProtocol.proto | 1 +
.../main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 1 +
.../ozone/protocolPB/OzoneManagerRequestHandler.java | 1 +
.../ozone/recon/metrics/OzoneManagerSyncMetrics.java | 12 ++++++++++++
.../recon/spi/impl/OzoneManagerServiceProviderImpl.java | 14 ++++++++++----
10 files changed, 52 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBUpdatesWrapper.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBUpdatesWrapper.java
index aa48c5e..d2dcc05 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBUpdatesWrapper.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBUpdatesWrapper.java
@@ -29,6 +29,7 @@ public class DBUpdatesWrapper {
private List<byte[]> dataList = new ArrayList<>();
private long currentSequenceNumber = -1;
+ private long latestSequenceNumber = -1;
public void addWriteBatch(byte[] data, long sequenceNumber) {
dataList.add(data);
@@ -48,5 +49,13 @@ public class DBUpdatesWrapper {
public long getCurrentSequenceNumber() {
return currentSequenceNumber;
}
+
+ public void setLatestSequenceNumber(long sequenceNumber) {
+ this.latestSequenceNumber = sequenceNumber;
+ }
+
+ public long getLatestSequenceNumber() {
+ return latestSequenceNumber;
+ }
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index eb71ec1..7e45994 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -432,6 +432,7 @@ public class RDBStore implements DBStore {
LOG.error("Unable to get delta updates since sequenceNumber {} ",
sequenceNumber, e);
}
+ dbUpdatesWrapper.setLatestSequenceNumber(db.getLatestSequenceNumber());
return dbUpdatesWrapper;
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/DBUpdates.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/DBUpdates.java
index 72f2b64..71916db 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/DBUpdates.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/DBUpdates.java
@@ -29,6 +29,8 @@ public class DBUpdates {
private long currentSequenceNumber = -1;
+ private long latestSequenceNumber = -1L;
+
public DBUpdates() {
this.dataList = new ArrayList<>();
}
@@ -55,4 +57,12 @@ public class DBUpdates {
public long getCurrentSequenceNumber() {
return currentSequenceNumber;
}
+
+ public void setLatestSequenceNumber(long sequenceNumber) {
+ this.latestSequenceNumber = sequenceNumber;
+ }
+
+ public long getLatestSequenceNumber() {
+ return latestSequenceNumber;
+ }
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 7750481..5883ff9 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -1515,6 +1515,8 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
}
dbUpdatesWrapper.setCurrentSequenceNumber(
dbUpdatesResponse.getSequenceNumber());
+ dbUpdatesWrapper.setLatestSequenceNumber(
+ dbUpdatesResponse.getLatestSequenceNumber());
return dbUpdatesWrapper;
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java
index ed903b3..701a799 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.recon.metrics.OzoneManagerSyncMetrics;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -201,6 +202,7 @@ public class TestReconWithOzoneManager {
OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl)
cluster.getReconServer().getOzoneManagerServiceProvider();
impl.syncDataFromOM();
+ OzoneManagerSyncMetrics metrics = impl.getMetrics();
// HTTP call to /api/containers
String containerResponse = makeHttpCall(containerKeyServiceURL);
@@ -229,6 +231,7 @@ public class TestReconWithOzoneManager {
// verify sequence number after full snapshot
Assert.assertEquals(omLatestSeqNumber, reconLatestSeqNumber);
+ Assert.assertEquals(0, metrics.getSequenceNumberLag().value());
//add 4 keys to check for delta updates
addKeys(1, 5);
@@ -264,6 +267,7 @@ public class TestReconWithOzoneManager {
//verify sequence number after Delta Updates
Assert.assertEquals(omLatestSeqNumber, reconLatestSeqNumber);
+ Assert.assertEquals(0, metrics.getSequenceNumberLag().value());
long beforeRestartSnapShotTimeStamp = getReconTaskAttributeFromJson(
taskStatusResponse,
@@ -321,6 +325,7 @@ public class TestReconWithOzoneManager {
//verify sequence number after Delta Updates
Assert.assertEquals(omLatestSeqNumber, reconLatestSeqNumber);
+ Assert.assertEquals(0, metrics.getSequenceNumberLag().value());
}
private long getReconTaskAttributeFromJson(String taskStatusResponse,
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index df3f2cc..2b539bc 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1115,6 +1115,7 @@ message ServiceListResponse {
message DBUpdatesResponse {
required uint64 sequenceNumber = 1;
repeated bytes data = 2;
+ optional uint64 latestSequenceNumber = 3;
}
message FinalizeUpgradeRequest {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index e2715af..16183da 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -3497,6 +3497,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
.getUpdatesSince(dbUpdatesRequest.getSequenceNumber(), limitCount);
DBUpdates dbUpdates = new DBUpdates(updatesSince.getData());
dbUpdates.setCurrentSequenceNumber(updatesSince.getCurrentSequenceNumber());
+ dbUpdates.setLatestSequenceNumber(updatesSince.getLatestSequenceNumber());
return dbUpdates;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 5674d6a..dbc63e3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -264,6 +264,7 @@ public class OzoneManagerRequestHandler implements RequestHandler {
dbUpdatesWrapper.getData().get(i)));
}
builder.setSequenceNumber(dbUpdatesWrapper.getCurrentSequenceNumber());
+ builder.setLatestSequenceNumber(dbUpdatesWrapper.getLatestSequenceNumber());
return builder.build();
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/OzoneManagerSyncMetrics.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/OzoneManagerSyncMetrics.java
index 1e3fa5a..e6ad328 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/OzoneManagerSyncMetrics.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/OzoneManagerSyncMetrics.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableGaugeFloat;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -75,6 +76,9 @@ public final class OzoneManagerSyncMetrics {
@Metric(about = "Average number of updates got per OM delta request")
private MutableGaugeFloat averageNumUpdatesInDeltaRequest;
+ @Metric(about = "The lag of sequence number between Recon and OM")
+ private MutableGaugeLong sequenceNumberLag;
+
public void incrNumSnapshotRequests() {
this.numSnapshotRequests.incr();
}
@@ -130,4 +134,12 @@ public final class OzoneManagerSyncMetrics {
public MutableCounterLong getNumNonZeroDeltaRequests() {
return numNonZeroDeltaRequests;
}
+
+ public void setSequenceNumberLag(long lag) {
+ sequenceNumberLag.set(lag);
+ }
+
+ public MutableGaugeLong getSequenceNumberLag() {
+ return sequenceNumberLag;
+ }
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
index ff73003..efebbf0 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
@@ -378,8 +378,8 @@ public class OzoneManagerServiceProviderImpl
long inLoopLatestSequenceNumber;
while (loopCount < deltaUpdateLoopLimit &&
deltaUpdateCnt >= deltaUpdateLimit) {
- innerGetAndApplyDeltaUpdatesFromOM(inLoopStartSequenceNumber,
- omdbUpdatesHandler);
+ innerGetAndApplyDeltaUpdatesFromOM(
+ inLoopStartSequenceNumber, omdbUpdatesHandler);
inLoopLatestSequenceNumber = getCurrentOMDBSequenceNumber();
deltaUpdateCnt = inLoopLatestSequenceNumber - inLoopStartSequenceNumber;
inLoopStartSequenceNumber = inLoopLatestSequenceNumber;
@@ -408,7 +408,9 @@ public class OzoneManagerServiceProviderImpl
.build();
DBUpdates dbUpdates = ozoneManagerClient.getDBUpdates(dbUpdatesRequest);
int numUpdates = 0;
+ long latestSequenceNumberOfOM = -1L;
if (null != dbUpdates && dbUpdates.getCurrentSequenceNumber() != -1) {
+ latestSequenceNumberOfOM = dbUpdates.getLatestSequenceNumber();
RDBStore rocksDBStore = (RDBStore) omMetadataManager.getStore();
RocksDB rocksDB = rocksDBStore.getDb();
numUpdates = dbUpdates.getData().size();
@@ -427,8 +429,12 @@ public class OzoneManagerServiceProviderImpl
}
}
}
- LOG.info("Number of updates received from OM : {}, SequenceNumber diff: {}",
- numUpdates, getCurrentOMDBSequenceNumber() - fromSequenceNumber);
+ long lag = latestSequenceNumberOfOM == -1 ? 0 :
+ latestSequenceNumberOfOM - getCurrentOMDBSequenceNumber();
+ metrics.setSequenceNumberLag(lag);
+ LOG.info("Number of updates received from OM : {}, " +
+ "SequenceNumber diff: {}, SequenceNumber Lag from OM {}.",
+ numUpdates, getCurrentOMDBSequenceNumber() - fromSequenceNumber, lag);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org