You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by av...@apache.org on 2021/03/08 01:07:40 UTC
[ozone] branch HDDS-3698-nonrolling-upgrade updated: HDDS-4902 Add
Layout version information to Recon datanode info API. (#1993)
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3698-nonrolling-upgrade by this push:
new 485ec25 HDDS-4902 Add Layout version information to Recon datanode info API. (#1993)
485ec25 is described below
commit 485ec255b9f70abc859b51abe2b4e856a64d0567
Author: avijayanhwx <14...@users.noreply.github.com>
AuthorDate: Sun Mar 7 17:07:16 2021 -0800
HDDS-4902 Add Layout version information to Recon datanode info API. (#1993)
---
.../hadoop/hdds/scm/node/SCMNodeManager.java | 58 +++++++++++-----------
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 2 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 2 +
.../hadoop/ozone/recon/api/NodeEndpoint.java | 5 ++
.../ozone/recon/api/types/DatanodeMetadata.java | 13 +++++
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 3 ++
7 files changed, 53 insertions(+), 32 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 5413bee..ea4a8cb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -540,37 +540,35 @@ public class SCMNodeManager implements NodeManager {
layoutVersionReport.toString().replaceAll("\n", "\\\\n"));
}
- if (layoutVersionReport != null) {
- int scmSlv = scmLayoutVersionManager.getSoftwareLayoutVersion();
- int scmMlv = scmLayoutVersionManager.getMetadataLayoutVersion();
- int dnSlv = layoutVersionReport.getSoftwareLayoutVersion();
- int dnMlv = layoutVersionReport.getMetadataLayoutVersion();
-
- // If the data node slv is > scm slv => log error condition
- if (dnSlv > scmSlv) {
- LOG.error("Rogue data node in the cluster : {}. " +
- "DataNode SoftwareLayoutVersion = {}, SCM " +
- "SoftwareLayoutVersion = {}",
- datanodeDetails.getHostName(), dnSlv, scmSlv);
- }
+ int scmSlv = scmLayoutVersionManager.getSoftwareLayoutVersion();
+ int scmMlv = scmLayoutVersionManager.getMetadataLayoutVersion();
+ int dnSlv = layoutVersionReport.getSoftwareLayoutVersion();
+ int dnMlv = layoutVersionReport.getMetadataLayoutVersion();
+
+ // If the data node slv is > scm slv => log error condition
+ if (dnSlv > scmSlv) {
+ LOG.error("Invalid data node in the cluster : {}. " +
+ "DataNode SoftwareLayoutVersion = {}, SCM " +
+ "SoftwareLayoutVersion = {}",
+ datanodeDetails.getHostName(), dnSlv, scmSlv);
+ }
- // If the datanode slv < scm slv, it can not be allowed to be part of
- // any pipeline. However it can be allowed to join the cluster
- if (dnMlv < scmMlv) {
- LOG.warn("Data node {} can not be used in any pipeline in the " +
- "cluster. " + "DataNode MetadataLayoutVersion = {}, SCM " +
- "MetadataLayoutVersion = {}",
- datanodeDetails.getHostName(), dnMlv, scmMlv);
-
- // Send Finalize command to the data node. Its OK to
- // send Finalize command multiple times.
- scmNodeEventPublisher.fireEvent(SCMEvents.DATANODE_COMMAND,
- new CommandForDatanode<>(datanodeDetails.getUuid(),
- new FinalizeNewLayoutVersionCommand(true,
- LayoutVersionProto.newBuilder()
- .setSoftwareLayoutVersion(dnSlv)
- .setMetadataLayoutVersion(dnSlv).build())));
- }
+ // If the datanode slv < scm slv, it can not be allowed to be part of
+ // any pipeline. However it can be allowed to join the cluster
+ if (dnMlv < scmMlv) {
+ LOG.warn("Data node {} can not be used in any pipeline in the " +
+ "cluster. " + "DataNode MetadataLayoutVersion = {}, SCM " +
+ "MetadataLayoutVersion = {}",
+ datanodeDetails.getHostName(), dnMlv, scmMlv);
+
+ // Send Finalize command to the data node. Its OK to
+ // send Finalize command multiple times.
+ scmNodeEventPublisher.fireEvent(SCMEvents.DATANODE_COMMAND,
+ new CommandForDatanode<>(datanodeDetails.getUuid(),
+ new FinalizeNewLayoutVersionCommand(true,
+ LayoutVersionProto.newBuilder()
+ .setSoftwareLayoutVersion(dnSlv)
+ .setMetadataLayoutVersion(dnSlv).build())));
}
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
index 2d02c3e..904d4f5 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@ -597,7 +597,7 @@ public class TestSCMNodeManager {
.setSoftwareLayoutVersion(scmMlv + 2)
.build());
Assert.assertTrue(logCapturer.getOutput()
- .contains("Rogue data node in the cluster"));
+ .contains("Invalid data node in the cluster"));
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java
index 6e3dfe3..eb21a81 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java
@@ -53,6 +53,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.Timeout;
+import org.slf4j.event.Level;
/**
* Recon's passive SCM integration tests.
@@ -79,6 +80,7 @@ public class TestReconAsPassiveScm {
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3)
.includeRecon(true).build();
cluster.waitForClusterToBeReady();
+ GenericTestUtils.setLogLevel(ReconNodeManager.LOG, Level.DEBUG);
}
@After
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java
index 2f88497..ce5b416 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.recon.api;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
@@ -119,6 +120,8 @@ public class NodeEndpoint {
LOG.warn("Cannot get containers, datanode {} not found.",
datanode.getUuid(), ex);
}
+
+ DatanodeInfo dnInfo = (DatanodeInfo) datanode;
datanodes.add(builder.withHostname(hostname)
.withDatanodeStorageReport(storageReport)
.withLastHeartbeat(nodeManager.getLastHeartbeat(datanode))
@@ -130,6 +133,8 @@ public class NodeEndpoint {
.withSetupTime(datanode.getSetupTime())
.withRevision(datanode.getRevision())
.withBuildDate(datanode.getBuildDate())
+ .withLayoutVersion(
+ dnInfo.getLastKnownLayoutVersion().getMetadataLayoutVersion())
.build());
});
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java
index f75ea32..426a388 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java
@@ -67,6 +67,9 @@ public final class DatanodeMetadata {
@XmlElement(name = "buildDate")
private String buildDate;
+ @XmlElement(name = "layoutVersion")
+ private int layoutVersion;
+
private DatanodeMetadata(Builder builder) {
this.hostname = builder.hostname;
this.uuid = builder.uuid;
@@ -80,6 +83,7 @@ public final class DatanodeMetadata {
this.setupTime = builder.setupTime;
this.revision = builder.revision;
this.buildDate = builder.buildDate;
+ this.layoutVersion = builder.layoutVersion;
}
public String getHostname() {
@@ -129,6 +133,9 @@ public final class DatanodeMetadata {
public String getBuildDate() {
return buildDate;
}
+ public int getLayoutVersion() {
+ return layoutVersion;
+ }
/**
* Returns new builder class that builds a DatanodeMetadata.
@@ -156,6 +163,7 @@ public final class DatanodeMetadata {
private long setupTime;
private String revision;
private String buildDate;
+ private int layoutVersion;
public Builder() {
this.containers = 0;
@@ -223,6 +231,11 @@ public final class DatanodeMetadata {
return this;
}
+ public Builder withLayoutVersion(int layoutVersion) {
+ this.layoutVersion = layoutVersion;
+ return this;
+ }
+
/**
* Constructs DatanodeMetadata.
*
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
index a659153..5af96b3 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
@@ -133,7 +133,7 @@ public class ReconNodeManager extends SCMNodeManager {
commandForDatanode.getCommand().getType())) {
super.onMessage(commandForDatanode, ignored);
} else {
- LOG.info("Ignoring unsupported command {} for Datanode {}.",
+ LOG.debug("Ignoring unsupported command {} for Datanode {}.",
commandForDatanode.getCommand().getType(),
commandForDatanode.getDatanodeId());
}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
index 02c42d9..9eabe06 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdds.utils.db.TypedTable;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -417,6 +418,8 @@ public class TestEndpoints extends AbstractReconSqlDBTest {
Assert.fail(String.format("Datanode %s not registered",
hostname));
}
+ Assert.assertEquals(HDDSLayoutVersionManager.maxLayoutVersion(),
+ datanodeMetadata.getLayoutVersion());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org