You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2021/02/08 12:24:59 UTC
[ozone] branch master updated: HDDS-4730. Use separate Ratis admin
and client ports (#1887)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 375da4d HDDS-4730. Use separate Ratis admin and client ports (#1887)
375da4d is described below
commit 375da4d231305c9e552a4c07ad0290f1ec89926e
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Mon Feb 8 13:24:44 2021 +0100
HDDS-4730. Use separate Ratis admin and client ports (#1887)
---
.../org/apache/hadoop/hdds/DatanodeVersions.java | 35 ++++++
.../hadoop/hdds/protocol/DatanodeDetails.java | 53 +++++++-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 36 +++---
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 12 ++
.../common/src/main/resources/ozone-default.xml | 14 ++-
.../apache/hadoop/ozone/HddsDatanodeService.java | 8 +-
.../container/common/helpers/DatanodeIdYaml.java | 38 +++++-
.../transport/server/ratis/XceiverServerRatis.java | 124 +++++++++++--------
.../src/main/compose/ozonesecure/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/restart/.env | 21 ++++
.../dist/src/main/compose/restart/README.md | 22 ++++
.../src/main/compose/restart/docker-compose.yaml | 134 +++++++++++++++++++++
.../dist/src/main/compose/restart/docker-config | 35 ++++++
hadoop-ozone/dist/src/main/compose/restart/test.sh | 54 +++++++++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 21 +++-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 66 +++++-----
16 files changed, 566 insertions(+), 108 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java
new file mode 100644
index 0000000..a8c5f1e
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds;
+
+/**
+ * Versioning for datanode.
+ */
+public final class DatanodeVersions {
+
+ public static final int DEFAULT_VERSION = 0;
+
+ public static final int SEPARATE_RATIS_PORTS_AVAILABLE = 1;
+
+ // this should always point to the latest version
+ public static final int CURRENT_VERSION = SEPARATE_RATIS_PORTS_AVAILABLE;
+
+ private DatanodeVersions() {
+ // no instances
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
index c8338b3..b98c116 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
@@ -25,6 +25,7 @@ import java.util.Set;
import java.util.UUID;
import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.hdds.DatanodeVersions;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
@@ -73,6 +74,8 @@ public class DatanodeDetails extends NodeImpl implements
private String buildDate;
private HddsProtos.NodeOperationalState persistedOpState;
private long persistedOpStateExpiryEpochSec = 0;
+ private int initialVersion;
+ private int currentVersion;
/**
* Constructs DatanodeDetails instance. DatanodeDetails.Builder is used
@@ -96,7 +99,8 @@ public class DatanodeDetails extends NodeImpl implements
String networkLocation, List<Port> ports, String certSerialId,
String version, long setupTime, String revision, String buildDate,
HddsProtos.NodeOperationalState persistedOpState,
- long persistedOpStateExpiryEpochSec) {
+ long persistedOpStateExpiryEpochSec,
+ int initialVersion, int currentVersion) {
super(hostName, networkLocation, NetConstants.NODE_COST_DEFAULT);
this.uuid = uuid;
this.uuidString = uuid.toString();
@@ -110,6 +114,8 @@ public class DatanodeDetails extends NodeImpl implements
this.buildDate = buildDate;
this.persistedOpState = persistedOpState;
this.persistedOpStateExpiryEpochSec = persistedOpStateExpiryEpochSec;
+ this.initialVersion = initialVersion;
+ this.currentVersion = currentVersion;
}
public DatanodeDetails(DatanodeDetails datanodeDetails) {
@@ -129,6 +135,8 @@ public class DatanodeDetails extends NodeImpl implements
this.persistedOpState = datanodeDetails.getPersistedOpState();
this.persistedOpStateExpiryEpochSec =
datanodeDetails.getPersistedOpStateExpiryEpochSec();
+ this.initialVersion = datanodeDetails.getInitialVersion();
+ this.currentVersion = datanodeDetails.getCurrentVersion();
}
/**
@@ -263,6 +271,10 @@ public class DatanodeDetails extends NodeImpl implements
return port;
}
}
+ // if no separate admin/server port, return single Ratis one for compat
+ if (name == Name.RATIS_ADMIN || name == Name.RATIS_SERVER) {
+ return getPort(Name.RATIS);
+ }
return null;
}
@@ -440,6 +452,28 @@ public class DatanodeDetails extends NodeImpl implements
return extendedBuilder.build();
}
+ /**
+ * @return the version this datanode was initially created with
+ */
+ public int getInitialVersion() {
+ return initialVersion;
+ }
+
+ public void setInitialVersion(int initialVersion) {
+ this.initialVersion = initialVersion;
+ }
+
+ /**
+ * @return the version this datanode was last started with
+ */
+ public int getCurrentVersion() {
+ return currentVersion;
+ }
+
+ public void setCurrentVersion(int currentVersion) {
+ this.currentVersion = currentVersion;
+ }
+
@Override
public String toString() {
return uuid.toString() + "{" +
@@ -498,6 +532,8 @@ public class DatanodeDetails extends NodeImpl implements
private String buildDate;
private HddsProtos.NodeOperationalState persistedOpState;
private long persistedOpStateExpiryEpochSec = 0;
+ private int initialVersion;
+ private int currentVersion = DatanodeVersions.CURRENT_VERSION;
/**
* Default private constructor. To create Builder instance use
@@ -683,6 +719,16 @@ public class DatanodeDetails extends NodeImpl implements
return this;
}
+ public Builder setInitialVersion(int v) {
+ this.initialVersion = v;
+ return this;
+ }
+
+ public Builder setCurrentVersion(int v) {
+ this.currentVersion = v;
+ return this;
+ }
+
/**
* Builds and returns DatanodeDetails instance.
*
@@ -695,7 +741,8 @@ public class DatanodeDetails extends NodeImpl implements
}
DatanodeDetails dn = new DatanodeDetails(id, ipAddress, hostName,
networkLocation, ports, certSerialId, version, setupTime, revision,
- buildDate, persistedOpState, persistedOpStateExpiryEpochSec);
+ buildDate, persistedOpState, persistedOpStateExpiryEpochSec,
+ initialVersion, currentVersion);
if (networkName != null) {
dn.setNetworkName(networkName);
}
@@ -724,7 +771,7 @@ public class DatanodeDetails extends NodeImpl implements
* Ports that are supported in DataNode.
*/
public enum Name {
- STANDALONE, RATIS, REST, REPLICATION;
+ STANDALONE, RATIS, REST, REPLICATION, RATIS_ADMIN, RATIS_SERVER;
public static final Set<Name> ALL_PORTS = ImmutableSet.copyOf(
Name.values());
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
index c910dd5..06acb44 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -31,6 +31,7 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
import org.apache.hadoop.hdds.ratis.retrypolicy.RetryPolicyCreator;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -40,9 +41,9 @@ import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.ratis.RaftConfigKeys;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.client.RaftClientConfigKeys;
+import org.apache.ratis.conf.Parameters;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
-import org.apache.ratis.grpc.GrpcFactory;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroup;
@@ -94,9 +95,8 @@ public final class RatisHelper {
return toDatanodeId(RaftPeerId.valueOf(peerId.getId()));
}
- private static String toRaftPeerAddressString(DatanodeDetails id) {
- return id.getIpAddress() + ":" +
- id.getPort(DatanodeDetails.Port.Name.RATIS).getValue();
+ private static String toRaftPeerAddress(DatanodeDetails id, Port.Name port) {
+ return id.getIpAddress() + ":" + id.getPort(port).getValue();
}
public static RaftPeerId toRaftPeerId(DatanodeDetails id) {
@@ -104,20 +104,23 @@ public final class RatisHelper {
}
public static RaftPeer toRaftPeer(DatanodeDetails id) {
- return RaftPeer.newBuilder()
- .setId(toRaftPeerId(id))
- .setAddress(toRaftPeerAddressString(id))
- .build();
+ return raftPeerBuilderFor(id).build();
}
public static RaftPeer toRaftPeer(DatanodeDetails id, int priority) {
- return RaftPeer.newBuilder()
- .setId(toRaftPeerId(id))
- .setAddress(toRaftPeerAddressString(id))
+ return raftPeerBuilderFor(id)
.setPriority(priority)
.build();
}
+ private static RaftPeer.Builder raftPeerBuilderFor(DatanodeDetails dn) {
+ return RaftPeer.newBuilder()
+ .setId(toRaftPeerId(dn))
+ .setAddress(toRaftPeerAddress(dn, Port.Name.RATIS_SERVER))
+ .setAdminAddress(toRaftPeerAddress(dn, Port.Name.RATIS_ADMIN))
+ .setClientAddress(toRaftPeerAddress(dn, Port.Name.RATIS));
+ }
+
private static List<RaftPeer> toRaftPeers(Pipeline pipeline) {
return toRaftPeers(pipeline.getNodes());
}
@@ -224,7 +227,9 @@ public final class RatisHelper {
// TODO: GRPC TLS only for now, netty/hadoop RPC TLS support later.
if (tlsConfig != null && rpcType == SupportedRpcType.GRPC) {
- builder.setParameters(GrpcFactory.newRaftParameters(tlsConfig));
+ Parameters parameters = new Parameters();
+ GrpcConfigKeys.Client.setTlsConf(parameters, tlsConfig);
+ builder.setParameters(parameters);
}
return builder.build();
}
@@ -254,9 +259,10 @@ public final class RatisHelper {
}
private static boolean isGrpcClientConfig(String key) {
- return key.startsWith(GrpcConfigKeys.PREFIX) && !key
- .startsWith(GrpcConfigKeys.TLS.PREFIX) && !key
- .startsWith(GrpcConfigKeys.Server.PREFIX);
+ return key.startsWith(GrpcConfigKeys.PREFIX) &&
+ !key.startsWith(GrpcConfigKeys.TLS.PREFIX) &&
+ !key.startsWith(GrpcConfigKeys.Admin.PREFIX) &&
+ !key.startsWith(GrpcConfigKeys.Server.PREFIX);
}
/**
* Set all server properties matching with prefix
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index c5c4426..873d288 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -64,6 +64,18 @@ public final class OzoneConfigKeys {
public static final String DFS_CONTAINER_RATIS_IPC_PORT =
"dfs.container.ratis.ipc";
public static final int DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT = 9858;
+ /**
+ * Ratis Port where containers listen to admin requests.
+ */
+ public static final String DFS_CONTAINER_RATIS_ADMIN_PORT =
+ "dfs.container.ratis.admin.port";
+ public static final int DFS_CONTAINER_RATIS_ADMIN_PORT_DEFAULT = 9857;
+ /**
+ * Ratis Port where containers listen to server-to-server requests.
+ */
+ public static final String DFS_CONTAINER_RATIS_SERVER_PORT =
+ "dfs.container.ratis.server.port";
+ public static final int DFS_CONTAINER_RATIS_SERVER_PORT_DEFAULT = 9856;
/**
* When set to true, allocate a random free port for ozone container, so that
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 47dbf39..9311937 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -167,8 +167,20 @@
<property>
<name>dfs.container.ratis.ipc</name>
<value>9858</value>
+ <tag>OZONE, CONTAINER, PIPELINE, RATIS</tag>
+ <description>The ipc port number of container for clients.</description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.admin.port</name>
+ <value>9857</value>
<tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag>
- <description>The ipc port number of container.</description>
+ <description>The ipc port number of container for admin requests.</description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.server.port</name>
+ <value>9856</value>
+ <tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag>
+ <description>The ipc port number of container for server-server communication.</description>
</property>
<property>
<name>dfs.container.ratis.ipc.random.port</name>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 0de14e6..25c4af7 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
+import org.apache.hadoop.hdds.DatanodeVersions;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.cli.GenericCli;
@@ -209,6 +210,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
datanodeDetails.setRevision(
HddsVersionInfo.HDDS_VERSION_INFO.getRevision());
datanodeDetails.setBuildDate(HddsVersionInfo.HDDS_VERSION_INFO.getDate());
+ datanodeDetails.setCurrentVersion(DatanodeVersions.CURRENT_VERSION);
TracingUtil.initTracing(
"HddsDatanodeService." + datanodeDetails.getUuidString()
.substring(0, 8), conf);
@@ -434,7 +436,11 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
} else {
// There is no datanode.id file, this might be the first time datanode
// is started.
- return DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build();
+ DatanodeDetails details = DatanodeDetails.newBuilder()
+ .setUuid(UUID.randomUUID()).build();
+ details.setInitialVersion(DatanodeVersions.CURRENT_VERSION);
+ details.setCurrentVersion(DatanodeVersions.CURRENT_VERSION);
+ return details;
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java
index 3b14641..52f5b3a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java
@@ -100,6 +100,10 @@ public final class DatanodeIdYaml {
portEntry.getValue()));
}
}
+
+ builder.setInitialVersion(datanodeDetailsYaml.getInitialVersion())
+ .setCurrentVersion(datanodeDetailsYaml.getCurrentVersion());
+
datanodeDetails = builder.build();
}
@@ -117,16 +121,19 @@ public final class DatanodeIdYaml {
private String persistedOpState;
private long persistedOpStateExpiryEpochSec = 0;
private Map<String, Integer> portDetails;
+ private int initialVersion;
+ private int currentVersion;
public DatanodeDetailsYaml() {
// Needed for snake-yaml introspection.
}
+ @SuppressWarnings({"parameternumber", "java:S107"}) // required for yaml
private DatanodeDetailsYaml(String uuid, String ipAddress,
- String hostName, String certSerialId,
- String persistedOpState,
- long persistedOpStateExpiryEpochSec,
- Map<String, Integer> portDetails) {
+ String hostName, String certSerialId,
+ String persistedOpState, long persistedOpStateExpiryEpochSec,
+ Map<String, Integer> portDetails,
+ int initialVersion, int currentVersion) {
this.uuid = uuid;
this.ipAddress = ipAddress;
this.hostName = hostName;
@@ -134,6 +141,8 @@ public final class DatanodeIdYaml {
this.persistedOpState = persistedOpState;
this.persistedOpStateExpiryEpochSec = persistedOpStateExpiryEpochSec;
this.portDetails = portDetails;
+ this.initialVersion = initialVersion;
+ this.currentVersion = currentVersion;
}
public String getUuid() {
@@ -191,6 +200,22 @@ public final class DatanodeIdYaml {
public void setPortDetails(Map<String, Integer> portDetails) {
this.portDetails = portDetails;
}
+
+ public int getInitialVersion() {
+ return initialVersion;
+ }
+
+ public void setInitialVersion(int version) {
+ this.initialVersion = version;
+ }
+
+ public int getCurrentVersion() {
+ return currentVersion;
+ }
+
+ public void setCurrentVersion(int version) {
+ this.currentVersion = version;
+ }
}
private static DatanodeDetailsYaml getDatanodeDetailsYaml(
@@ -207,6 +232,7 @@ public final class DatanodeIdYaml {
if (datanodeDetails.getPersistedOpState() != null) {
persistedOpString = datanodeDetails.getPersistedOpState().name();
}
+
return new DatanodeDetailsYaml(
datanodeDetails.getUuid().toString(),
datanodeDetails.getIpAddress(),
@@ -214,6 +240,8 @@ public final class DatanodeIdYaml {
datanodeDetails.getCertSerialId(),
persistedOpString,
datanodeDetails.getPersistedOpStateExpiryEpochSec(),
- portDetails);
+ portDetails,
+ datanodeDetails.getInitialVersion(),
+ datanodeDetails.getCurrentVersion());
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index dec60f4..09d1ed0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import java.io.File;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -45,6 +46,7 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto;
@@ -77,9 +79,9 @@ import io.opentracing.Scope;
import io.opentracing.Span;
import io.opentracing.util.GlobalTracer;
import org.apache.ratis.RaftConfigKeys;
+import org.apache.ratis.conf.Parameters;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
-import org.apache.ratis.grpc.GrpcFactory;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.netty.NettyConfigKeys;
import org.apache.ratis.proto.RaftProtos;
@@ -100,12 +102,14 @@ import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
+import org.apache.ratis.server.RaftServerRpc;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.TimeDuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.hdds.DatanodeVersions.SEPARATE_RATIS_PORTS_AVAILABLE;
/**
* Creates a ratis server endpoint that acts as the communication layer for
@@ -123,7 +127,9 @@ public final class XceiverServerRatis implements XceiverServerSpi {
return CALL_ID_COUNTER.getAndIncrement() & Long.MAX_VALUE;
}
- private int port;
+ private int serverPort;
+ private int adminPort;
+ private int clientPort;
private final RaftServer server;
private final List<ThreadPoolExecutor> chunkExecutors;
private final ContainerDispatcher dispatcher;
@@ -149,14 +155,14 @@ public final class XceiverServerRatis implements XceiverServerSpi {
*/
private EnumMap<StorageType, List<String>> ratisVolumeMap;
- private XceiverServerRatis(DatanodeDetails dd, int port,
+ private XceiverServerRatis(DatanodeDetails dd,
ContainerDispatcher dispatcher, ContainerController containerController,
- StateContext context, GrpcTlsConfig tlsConfig, ConfigurationSource conf)
+ StateContext context, ConfigurationSource conf, Parameters parameters)
throws IOException {
this.conf = conf;
Objects.requireNonNull(dd, "id == null");
datanodeDetails = dd;
- this.port = port;
+ assignPorts();
RaftProperties serverProperties = newRaftProperties();
this.context = context;
this.dispatcher = dispatcher;
@@ -173,10 +179,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftServer.Builder builder =
RaftServer.newBuilder().setServerId(raftPeerId)
.setProperties(serverProperties)
- .setStateMachineRegistry(this::getStateMachine);
- if (tlsConfig != null) {
- builder.setParameters(GrpcFactory.newRaftParameters(tlsConfig));
- }
+ .setStateMachineRegistry(this::getStateMachine)
+ .setParameters(parameters);
this.server = builder.build();
this.requestTimeout = conf.getTimeDuration(
HddsConfigKeys.HDDS_DATANODE_RATIS_SERVER_REQUEST_TIMEOUT,
@@ -185,6 +189,31 @@ public final class XceiverServerRatis implements XceiverServerSpi {
initializeRatisVolumeMap();
}
+ private void assignPorts() {
+ clientPort = determinePort(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT);
+
+ if (datanodeDetails.getInitialVersion() >= SEPARATE_RATIS_PORTS_AVAILABLE) {
+ adminPort = determinePort(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_ADMIN_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_ADMIN_PORT_DEFAULT);
+ serverPort = determinePort(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SERVER_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SERVER_PORT_DEFAULT);
+ } else {
+ adminPort = clientPort;
+ serverPort = clientPort;
+ }
+ }
+
+ private int determinePort(String key, int defaultValue) {
+ boolean randomPort = conf.getBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT);
+ return randomPort ? 0 : conf.getInt(key, defaultValue);
+ }
+
private ContainerStateMachine getStateMachine(RaftGroupId gid) {
return new ContainerStateMachine(gid, dispatcher, containerController,
chunkExecutors, this, conf);
@@ -262,9 +291,11 @@ public final class XceiverServerRatis implements XceiverServerSpi {
// Set the ratis port number
if (rpc == SupportedRpcType.GRPC) {
- GrpcConfigKeys.Server.setPort(properties, port);
+ GrpcConfigKeys.Admin.setPort(properties, adminPort);
+ GrpcConfigKeys.Client.setPort(properties, clientPort);
+ GrpcConfigKeys.Server.setPort(properties, serverPort);
} else if (rpc == SupportedRpcType.NETTY) {
- NettyConfigKeys.Server.setPort(properties, port);
+ NettyConfigKeys.Server.setPort(properties, serverPort);
}
long snapshotThreshold =
@@ -416,71 +447,68 @@ public final class XceiverServerRatis implements XceiverServerSpi {
DatanodeDetails datanodeDetails, ConfigurationSource ozoneConf,
ContainerDispatcher dispatcher, ContainerController containerController,
CertificateClient caClient, StateContext context) throws IOException {
- int localPort = ozoneConf.getInt(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT);
+ Parameters parameters = createTlsParameters(
+ new SecurityConfig(ozoneConf), caClient);
- // Get an available port on current node and
- // use that as the container port
- if (ozoneConf.getBoolean(OzoneConfigKeys
- .DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT)) {
- localPort = 0;
- }
- GrpcTlsConfig tlsConfig = createTlsServerConfigForDN(
- new SecurityConfig(ozoneConf), caClient);
-
- return new XceiverServerRatis(datanodeDetails, localPort, dispatcher,
- containerController, context, tlsConfig, ozoneConf);
+ return new XceiverServerRatis(datanodeDetails, dispatcher,
+ containerController, context, ozoneConf, parameters);
}
// For gRPC server running DN container service with gPRC TLS
- // No mTLS as the channel is shared for for external client, which
- // does not have SCM CA issued certificates.
// In summary:
// authenticate from server to client is via TLS.
// authenticate from client to server is via block token (or container token).
// DN Ratis server act as both SSL client and server and we must pass TLS
// configuration for both.
- static GrpcTlsConfig createTlsServerConfigForDN(SecurityConfig conf,
+ private static Parameters createTlsParameters(SecurityConfig conf,
CertificateClient caClient) {
+ Parameters parameters = new Parameters();
+
if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) {
- return new GrpcTlsConfig(
+ GrpcTlsConfig serverConfig = new GrpcTlsConfig(
+ caClient.getPrivateKey(), caClient.getCertificate(),
+ caClient.getCACertificate(), true);
+ GrpcConfigKeys.Server.setTlsConf(parameters, serverConfig);
+ GrpcConfigKeys.Admin.setTlsConf(parameters, serverConfig);
+
+ GrpcTlsConfig clientConfig = new GrpcTlsConfig(
caClient.getPrivateKey(), caClient.getCertificate(),
caClient.getCACertificate(), false);
+ GrpcConfigKeys.Client.setTlsConf(parameters, clientConfig);
}
- return null;
+
+ return parameters;
}
@Override
public void start() throws IOException {
if (!isStarted) {
- LOG.info("Starting {} {} at port {}", getClass().getSimpleName(),
- server.getId(), getIPCPort());
+ LOG.info("Starting {} {}", getClass().getSimpleName(), server.getId());
for (ThreadPoolExecutor executor : chunkExecutors) {
executor.prestartAllCoreThreads();
}
server.start();
- int realPort = server.getServerRpc()
- .getInetSocketAddress()
- .getPort();
-
- if (port == 0) {
- LOG.info("{} {} is started using port {}", getClass().getSimpleName(),
- server.getId(), realPort);
- port = realPort;
- }
-
- //register the real port to the datanode details.
- datanodeDetails.setPort(DatanodeDetails
- .newPort(DatanodeDetails.Port.Name.RATIS,
- realPort));
+ RaftServerRpc serverRpc = server.getServerRpc();
+ clientPort = getRealPort(serverRpc.getClientServerAddress(),
+ Port.Name.RATIS);
+ adminPort = getRealPort(serverRpc.getAdminServerAddress(),
+ Port.Name.RATIS_ADMIN);
+ serverPort = getRealPort(serverRpc.getInetSocketAddress(),
+ Port.Name.RATIS_SERVER);
isStarted = true;
}
}
+ private int getRealPort(InetSocketAddress address, Port.Name name) {
+ int realPort = address.getPort();
+ datanodeDetails.setPort(DatanodeDetails.newPort(name, realPort));
+ LOG.info("{} {} is started using port {} for {}",
+ getClass().getSimpleName(), server.getId(), realPort, name);
+ return realPort;
+ }
+
@Override
public void stop() {
if (isStarted) {
@@ -500,7 +528,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
@Override
public int getIPCPort() {
- return port;
+ return clientPort;
}
/**
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index ec94dfb..603427e 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -29,6 +29,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_hdds.block.token.enabled=true
+OZONE-SITE.XML_hdds.grpc.tls.enabled=true
OZONE-SITE.XML_ozone.replication=3
OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
diff --git a/hadoop-ozone/dist/src/main/compose/restart/.env b/hadoop-ozone/dist/src/main/compose/restart/.env
new file mode 100644
index 0000000..6f757c5
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/restart/.env
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
+OZONE_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
+OZONE_DIR=/opt/hadoop
+OZONE_VOLUME=.
diff --git a/hadoop-ozone/dist/src/main/compose/restart/README.md b/hadoop-ozone/dist/src/main/compose/restart/README.md
new file mode 100644
index 0000000..8bb4213
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/restart/README.md
@@ -0,0 +1,22 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Compose file for restart
+
+This directory contains a sample cluster definition and script for
+testing restart of the cluster with the same version.
+
+Data for each container is persisted in mounted volume (by default it's
+`data` under the `compose/restart` directory, but can be overridden via
+`OZONE_VOLUME` environment variable).
diff --git a/hadoop-ozone/dist/src/main/compose/restart/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/restart/docker-compose.yaml
new file mode 100644
index 0000000..a15c2f9
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/restart/docker-compose.yaml
@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields)
+x-common-config:
+ &common-config
+ env_file:
+ - docker-config
+ image: ${OZONE_IMAGE}
+
+x-replication:
+ &replication
+ OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1}
+
+x-datanode:
+ &datanode
+ command: ["ozone","datanode"]
+ <<: *common-config
+ environment:
+ <<: *replication
+ ports:
+ - 9864
+ - 9882
+
+services:
+ dn1:
+ <<: *datanode
+ networks:
+ net:
+ ipv4_address: 10.9.0.11
+ volumes:
+ - ${OZONE_VOLUME}/dn1:/data
+ - ../..:${OZONE_DIR}
+ - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+ dn2:
+ <<: *datanode
+ networks:
+ net:
+ ipv4_address: 10.9.0.12
+ volumes:
+ - ${OZONE_VOLUME}/dn2:/data
+ - ../..:${OZONE_DIR}
+ - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+ dn3:
+ <<: *datanode
+ networks:
+ net:
+ ipv4_address: 10.9.0.13
+ volumes:
+ - ${OZONE_VOLUME}/dn3:/data
+ - ../..:${OZONE_DIR}
+ - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+ om:
+ command: ["ozone","om"]
+ <<: *common-config
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ <<: *replication
+ networks:
+ net:
+ ipv4_address: 10.9.0.14
+ ports:
+ - 9874:9874
+ - 9862:9862
+ volumes:
+ - ${OZONE_VOLUME}/om:/data
+ - ../..:${OZONE_DIR}
+ - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+ recon:
+ command: ["ozone","recon"]
+ <<: *common-config
+ environment:
+ <<: *replication
+ networks:
+ net:
+ ipv4_address: 10.9.0.15
+ ports:
+ - 9888:9888
+ volumes:
+ - ${OZONE_VOLUME}/recon:/data
+ - ../..:${OZONE_DIR}
+ - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+ s3g:
+ command: ["ozone","s3g"]
+ <<: *common-config
+ environment:
+ <<: *replication
+ networks:
+ net:
+ ipv4_address: 10.9.0.16
+ ports:
+ - 9878:9878
+ volumes:
+ - ${OZONE_VOLUME}/s3g:/data
+ - ../..:${OZONE_DIR}
+ - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+ scm:
+ command: ["ozone","scm"]
+ <<: *common-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
+ <<: *replication
+ networks:
+ net:
+ ipv4_address: 10.9.0.17
+ ports:
+ - 9876:9876
+ volumes:
+ - ${OZONE_VOLUME}/scm:/data
+ - ../..:${OZONE_DIR}
+ - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+
+networks:
+ net:
+ driver: bridge
+ ipam:
+ config:
+ - subnet: 10.9.0.0/16
diff --git a/hadoop-ozone/dist/src/main/compose/restart/docker-config b/hadoop-ozone/dist/src/main/compose/restart/docker-config
new file mode 100644
index 0000000..83530bb
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/restart/docker-config
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
+OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_ozone.recon.address=recon:9891
+OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
+
+OZONE_CONF_DIR=/etc/hadoop
+OZONE_LOG_DIR=/var/log/hadoop
+
+no_proxy=om,scm,s3g,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/restart/test.sh b/hadoop-ozone/dist/src/main/compose/restart/test.sh
new file mode 100644
index 0000000..41e06c5
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/restart/test.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+: "${OZONE_REPLICATION_FACTOR:=3}"
+: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
+
+export OZONE_VOLUME
+
+mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
+
+if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
+ current_user=$(whoami)
+ if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
+ chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
+ || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
+ fi
+fi
+
+# shellcheck source=/dev/null
+source "${COMPOSE_DIR}/../testlib.sh"
+
+# prepare pre-upgrade cluster
+start_docker_env
+execute_robot_test scm -v PREFIX:pre freon/generate.robot
+execute_robot_test scm -v PREFIX:pre freon/validate.robot
+KEEP_RUNNING=false stop_docker_env
+
+# re-start cluster with new version and check after upgrade
+export OZONE_KEEP_RESULTS=true
+start_docker_env
+execute_robot_test scm -v PREFIX:pre freon/validate.robot
+# test write key to old bucket after upgrade
+execute_robot_test scm -v PREFIX:post freon/generate.robot
+execute_robot_test scm -v PREFIX:post freon/validate.robot
+stop_docker_env
+
+generate_report
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index c37552f..4e046ab 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -70,13 +70,19 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.commons.io.FileUtils;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
+import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.RATIS;
+import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.RATIS_ADMIN;
+import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.RATIS_SERVER;
+import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.STANDALONE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY;
import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_IPC_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ADMIN_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_SERVER_PORT;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
@@ -345,14 +351,17 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
stopDatanode(datanodeService);
// ensure same ports are used across restarts.
OzoneConfiguration config = datanodeService.getConf();
- int currentPort = datanodeService.getDatanodeDetails()
- .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
- config.setInt(DFS_CONTAINER_IPC_PORT, currentPort);
+ DatanodeDetails dn = datanodeService.getDatanodeDetails();
config.setBoolean(DFS_CONTAINER_IPC_RANDOM_PORT, false);
- int ratisPort = datanodeService.getDatanodeDetails()
- .getPort(DatanodeDetails.Port.Name.RATIS).getValue();
- config.setInt(DFS_CONTAINER_RATIS_IPC_PORT, ratisPort);
config.setBoolean(DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, false);
+ config.setInt(DFS_CONTAINER_IPC_PORT,
+ dn.getPort(STANDALONE).getValue());
+ config.setInt(DFS_CONTAINER_RATIS_IPC_PORT,
+ dn.getPort(RATIS).getValue());
+ config.setInt(DFS_CONTAINER_RATIS_ADMIN_PORT,
+ dn.getPort(RATIS_ADMIN).getValue());
+ config.setInt(DFS_CONTAINER_RATIS_SERVER_PORT,
+ dn.getPort(RATIS_SERVER).getValue());
hddsDatanodes.remove(i);
if (waitForDatanode) {
// wait for node to be removed from SCM healthy node list.
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index 5ff02ad..178ea6d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -139,34 +139,12 @@ public class TestMiniOzoneCluster {
public void testDatanodeIDPersistent() throws Exception {
// Generate IDs for testing
DatanodeDetails id1 = randomDatanodeDetails();
- DatanodeDetails id2 = randomDatanodeDetails();
- DatanodeDetails id3 = randomDatanodeDetails();
id1.setPort(DatanodeDetails.newPort(Port.Name.STANDALONE, 1));
- id2.setPort(DatanodeDetails.newPort(Port.Name.STANDALONE, 2));
- id3.setPort(DatanodeDetails.newPort(Port.Name.STANDALONE, 3));
+ assertWriteRead(id1);
// Add certificate serial id.
- String certSerialId = "" + RandomUtils.nextLong();
- id1.setCertSerialId(certSerialId);
-
- // Write a single ID to the file and read it out
- File validIdsFile = new File(WRITE_TMP, "valid-values.id");
- validIdsFile.delete();
- ContainerUtils.writeDatanodeDetailsTo(id1, validIdsFile);
- // Validate using yaml parser
- Yaml yaml = new Yaml();
- try {
- yaml.load(new InputStreamReader(new FileInputStream(validIdsFile),
- StandardCharsets.UTF_8));
- } catch (Exception e) {
- Assert.fail("Failed parsing datanode id yaml.");
- }
- DatanodeDetails validId = ContainerUtils.readDatanodeDetailsFrom(
- validIdsFile);
-
- assertEquals(validId.getCertSerialId(), certSerialId);
- assertEquals(id1, validId);
- assertEquals(id1.getProtoBufMessage(), validId.getProtoBufMessage());
+ id1.setCertSerialId("" + RandomUtils.nextLong());
+ assertWriteRead(id1);
// Read should return an empty value if file doesn't exist
File nonExistFile = new File(READ_TMP, "non_exist.id");
@@ -194,10 +172,32 @@ public class TestMiniOzoneCluster {
HddsProtos.DatanodeDetailsProto proto = id1.getProtoBufMessage();
proto.writeTo(out);
}
- validId = ContainerUtils.readDatanodeDetailsFrom(protoFile);
- assertEquals(validId.getCertSerialId(), certSerialId);
- assertEquals(id1, validId);
- assertEquals(id1.getProtoBufMessage(), validId.getProtoBufMessage());
+ assertDetailsEquals(id1, ContainerUtils.readDatanodeDetailsFrom(protoFile));
+
+ id1.setInitialVersion(1);
+ assertWriteRead(id1);
+ }
+
+ private static void assertWriteRead(DatanodeDetails details)
+ throws IOException {
+ // Write a single ID to the file and read it out
+ File file = new File(WRITE_TMP, "valid-values.id");
+ file.delete();
+ ContainerUtils.writeDatanodeDetailsTo(details, file);
+
+ // Validate using yaml parser
+ Yaml yaml = new Yaml();
+ try {
+ yaml.load(new InputStreamReader(new FileInputStream(file),
+ StandardCharsets.UTF_8));
+ } catch (Exception e) {
+ Assert.fail("Failed parsing datanode id yaml.");
+ }
+
+ DatanodeDetails read = ContainerUtils.readDatanodeDetailsFrom(file);
+
+ assertDetailsEquals(details, read);
+ assertEquals(details.getCurrentVersion(), read.getCurrentVersion());
}
@Test
@@ -360,4 +360,12 @@ public class TestMiniOzoneCluster {
.getDatanodeStateMachine().getContainer().getVolumeSet()
.getVolumesList().size());
}
+
+ private static void assertDetailsEquals(DatanodeDetails expected,
+ DatanodeDetails actual) {
+ assertEquals(expected, actual);
+ assertEquals(expected.getCertSerialId(), actual.getCertSerialId());
+ assertEquals(expected.getProtoBufMessage(), actual.getProtoBufMessage());
+ assertEquals(expected.getInitialVersion(), actual.getInitialVersion());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org