You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by bh...@apache.org on 2021/02/22 03:39:01 UTC
[ozone] branch HDDS-2823 updated: HDDS-4821. Use SCM service ID in
SCMBlockClient and SCM Client. (#1940)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-2823 by this push:
new 8d49b81 HDDS-4821. Use SCM service ID in SCMBlockClient and SCM Client. (#1940)
8d49b81 is described below
commit 8d49b8190fafaab9022307678a1b82e9e05276ac
Author: Bharat Viswanadham <bh...@apache.org>
AuthorDate: Mon Feb 22 09:08:46 2021 +0530
HDDS-4821. Use SCM service ID in SCMBlockClient and SCM Client. (#1940)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 0
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 36 ++-
.../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 246 +++++++++++++++++++++
.../apache/hadoop/hdds/scm/ha/package-info.java | 21 ++
.../org/apache/hadoop/hdds/server/ServerUtils.java | 0
.../apache/hadoop/hdds/server/package-info.java | 23 ++
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 8 +-
.../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 0
.../org/apache/hadoop/ozone/ha/package-info.java | 21 ++
.../common/src/main/resources/ozone-default.xml | 2 +-
.../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 163 ++++++++++++++
.../apache/hadoop/hdds/scm/ha/package-info.java | 21 ++
.../SCMBlockLocationFailoverProxyProvider.java | 100 +++------
.../SCMContainerLocationFailoverProxyProvider.java | 108 +++------
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 6 +-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 65 +++---
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 8 +-
18 files changed, 647 insertions(+), 185 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 17fb7a0..d0f48bf 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -277,8 +277,8 @@ public final class ScmConfigKeys {
// able to send back a new list to the datanodes.
public static final String OZONE_SCM_NAMES = "ozone.scm.names";
- public static final String OZONE_SCM_INTERNAL_SERVICE_ID =
- "ozone.scm.internal.service.id";
+ public static final String OZONE_SCM_DEFAULT_SERVICE_ID =
+ "ozone.scm.default.service.id";
public static final String OZONE_SCM_SERVICE_IDS_KEY =
"ozone.scm.service.ids";
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java
similarity index 100%
rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
similarity index 73%
rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
index b61023f..3a6ef47 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
@@ -18,8 +18,9 @@
package org.apache.hadoop.hdds.scm.ha;
+
import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.server.ServerUtils;
@@ -29,9 +30,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.nio.file.Paths;
import java.util.Collection;
-
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_DIR;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY;
/**
* Utility class used by SCM HA.
@@ -52,7 +54,7 @@ public final class SCMHAUtils {
/**
* Get a collection of all scmNodeIds for the given scmServiceId.
*/
- public static Collection<String> getSCMNodeIds(Configuration conf,
+ public static Collection<String> getSCMNodeIds(ConfigurationSource conf,
String scmServiceId) {
String key = addSuffix(ScmConfigKeys.OZONE_SCM_NODES_KEY, scmServiceId);
return conf.getTrimmedStringCollection(key);
@@ -101,4 +103,32 @@ public final class SCMHAUtils {
}
return snapshotDir;
}
+
+ /**
+ * Get SCM ServiceId from OzoneConfiguration.
+ * @param conf
+ * @return SCM service id if defined, else null.
+ */
+ public static String getScmServiceId(ConfigurationSource conf) {
+
+ String localScmServiceId = conf.getTrimmed(
+ ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID);
+
+ Collection< String > scmServiceIds;
+
+ if (localScmServiceId == null) {
+ // There is no default scm service id is being set, fall back to ozone
+ // .scm.service.ids.
+ scmServiceIds = conf.getTrimmedStringCollection(
+ OZONE_SCM_SERVICE_IDS_KEY);
+ if (scmServiceIds.size() > 1) {
+ throw new ConfigurationException("When multiple SCM Service Ids are " +
+ "configured," + OZONE_SCM_DEFAULT_SERVICE_ID + " need to be " +
+ "defined");
+ } else if (scmServiceIds.size() == 1) {
+ localScmServiceId = scmServiceIds.iterator().next();
+ }
+ }
+ return localScmServiceId;
+ }
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java
new file mode 100644
index 0000000..3c6a680
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.ha;
+
+import org.apache.hadoop.hdds.conf.ConfigurationException;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.OptionalInt;
+
+import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
+import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NODES_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.SCM_DUMMY_NODEID;
+import static org.apache.hadoop.ozone.OzoneConsts.SCM_DUMMY_SERVICE_ID;
+
+/**
+ * SCM Node Info.
+ */
+public class SCMNodeInfo {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SCMNodeInfo.class);
+ private String serviceId;
+ private String nodeId;
+ private String blockClientAddress;
+ private String scmClientAddress;
+ private String scmSecurityAddress;
+ private String scmDatanodeAddress;
+
+ /**
+ * Build SCM Node information from configuration.
+ * @param conf
+ * @return
+ */
+ public static List<SCMNodeInfo> buildNodeInfo(ConfigurationSource conf) {
+
+ // First figure out scm client address from HA style config.
+ // If service Id is not defined, fall back to non-HA config.
+
+ List<SCMNodeInfo> scmNodeInfoList = new ArrayList<>();
+ String scmServiceId = SCMHAUtils.getScmServiceId(conf);
+ if (scmServiceId != null) {
+ ArrayList< String > scmNodeIds = new ArrayList<>(
+ SCMHAUtils.getSCMNodeIds(conf, scmServiceId));
+ if (scmNodeIds.size() == 0) {
+ throw new ConfigurationException(
+ String.format("Configuration does not have any value set for %s " +
+ "for the SCM serviceId %s. List of SCM Node ID's should " +
+ "be specified for an SCM HA service", OZONE_SCM_NODES_KEY,
+ scmServiceId));
+ }
+
+ for (String scmNodeId : scmNodeIds) {
+ String addressKey = ConfUtils.addKeySuffixes(
+ OZONE_SCM_ADDRESS_KEY, scmServiceId, scmNodeId);
+ String scmAddress = conf.get(addressKey);
+ if (scmAddress == null) {
+ throw new ConfigurationException(addressKey + "is not defined");
+ }
+
+ // Get port from Address Key if defined, else fall back to port key.
+ int scmClientPort = getPort(conf, scmServiceId, scmNodeId,
+ OZONE_SCM_CLIENT_ADDRESS_KEY, OZONE_SCM_CLIENT_PORT_KEY,
+ OZONE_SCM_CLIENT_PORT_DEFAULT);
+
+ int scmBlockClientPort = getPort(conf, scmServiceId, scmNodeId,
+ OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+ OZONE_SCM_BLOCK_CLIENT_PORT_KEY,
+ OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT);
+
+ int scmSecurityPort = getPort(conf, scmServiceId, scmNodeId,
+ OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY,
+ OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+ OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT);
+
+ int scmDatanodePort = getPort(conf, scmServiceId, scmNodeId,
+ OZONE_SCM_DATANODE_ADDRESS_KEY, OZONE_SCM_DATANODE_PORT_KEY,
+ OZONE_SCM_DATANODE_PORT_DEFAULT);
+
+ scmNodeInfoList.add(new SCMNodeInfo(scmServiceId, scmNodeId,
+ buildAddress(scmAddress, scmBlockClientPort),
+ buildAddress(scmAddress, scmClientPort),
+ buildAddress(scmAddress, scmSecurityPort),
+ buildAddress(scmAddress, scmDatanodePort)));
+ }
+ return scmNodeInfoList;
+ } else {
+ scmServiceId = SCM_DUMMY_SERVICE_ID;
+
+ // Following current approach of fall back to
+ // OZONE_SCM_CLIENT_ADDRESS_KEY to figure out hostname.
+
+ String scmBlockClientAddress = getHostNameFromConfigKeys(conf,
+ OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+ OZONE_SCM_CLIENT_ADDRESS_KEY).orElse(null);
+
+ String scmClientAddress = getHostNameFromConfigKeys(conf,
+ OZONE_SCM_CLIENT_ADDRESS_KEY).orElse(null);
+
+ String scmSecurityClientAddress =
+ getHostNameFromConfigKeys(conf,
+ OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY,
+ OZONE_SCM_CLIENT_ADDRESS_KEY).orElse(null);
+
+ String scmDatanodeAddress =
+ getHostNameFromConfigKeys(conf,
+ OZONE_SCM_DATANODE_ADDRESS_KEY,
+ OZONE_SCM_CLIENT_ADDRESS_KEY).orElse(null);
+
+ int scmBlockClientPort = getPortNumberFromConfigKeys(conf,
+ OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY)
+ .orElse(conf.getInt(OZONE_SCM_BLOCK_CLIENT_PORT_KEY,
+ OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
+
+ int scmClientPort = getPortNumberFromConfigKeys(conf,
+ OZONE_SCM_CLIENT_ADDRESS_KEY)
+ .orElse(conf.getInt(OZONE_SCM_CLIENT_PORT_KEY,
+ OZONE_SCM_CLIENT_PORT_DEFAULT));
+
+ int scmSecurityPort = getPortNumberFromConfigKeys(conf,
+ OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY)
+ .orElse(conf.getInt(OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+ OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT));
+
+ int scmDatanodePort = getPortNumberFromConfigKeys(conf,
+ OZONE_SCM_DATANODE_ADDRESS_KEY)
+ .orElse(conf.getInt(OZONE_SCM_DATANODE_PORT_KEY,
+ OZONE_SCM_DATANODE_PORT_DEFAULT));
+
+ scmNodeInfoList.add(new SCMNodeInfo(scmServiceId,
+ SCM_DUMMY_NODEID,
+ scmBlockClientAddress == null ? null :
+ buildAddress(scmBlockClientAddress, scmBlockClientPort),
+ scmClientAddress == null ? null :
+ buildAddress(scmClientAddress, scmClientPort),
+ scmSecurityClientAddress == null ? null :
+ buildAddress(scmSecurityClientAddress, scmSecurityPort),
+ scmDatanodeAddress == null ? null :
+ buildAddress(scmDatanodeAddress, scmDatanodePort)));
+
+ return scmNodeInfoList;
+
+ }
+
+ }
+
+ private static String buildAddress(String address, int port) {
+ return new StringBuilder().append(address).append(":")
+ .append(port).toString();
+ }
+
+ private static int getPort(ConfigurationSource conf,
+ String scmServiceId, String scmNodeId, String configKey,
+ String portKey, int defaultPort) {
+ String suffixKey = ConfUtils.addKeySuffixes(configKey, scmServiceId,
+ scmNodeId);
+ OptionalInt port = getPortNumberFromConfigKeys(conf, suffixKey);
+
+ if (port.isPresent()) {
+ LOG.info("ConfigKey {} is deprecated, For configuring different " +
+ "ports for each SCM use PortConfigKey {} appended with serviceId " +
+ "and nodeId", configKey, portKey);
+ return port.getAsInt();
+ } else {
+ return conf.getInt(ConfUtils.addKeySuffixes(portKey, scmServiceId,
+ scmNodeId), defaultPort);
+ }
+ }
+
+ /**
+ * SCM Node Info which contains information about scm service address.
+ * @param serviceId
+ * @param nodeId
+ * @param blockClientAddress
+ * @param scmClientAddress
+ * @param scmSecurityAddress
+ * @param scmDatanodeAddress
+ */
+ public SCMNodeInfo(String serviceId, String nodeId,
+ String blockClientAddress, String scmClientAddress,
+ String scmSecurityAddress, String scmDatanodeAddress) {
+ this.serviceId = serviceId;
+ this.nodeId = nodeId;
+ this.blockClientAddress = blockClientAddress;
+ this.scmClientAddress = scmClientAddress;
+ this.scmSecurityAddress = scmSecurityAddress;
+ this.scmDatanodeAddress = scmDatanodeAddress;
+ }
+
+ public String getServiceId() {
+ return serviceId;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public String getBlockClientAddress() {
+ return blockClientAddress;
+ }
+
+ public String getScmClientAddress() {
+ return scmClientAddress;
+ }
+
+ public String getScmSecurityAddress() {
+ return scmSecurityAddress;
+ }
+
+ public String getScmDatanodeAddress() {
+ return scmDatanodeAddress;
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/package-info.java
new file mode 100644
index 0000000..19153b0
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.ha;
+/**
+ Utility slasses for SCM HA.
+ */
\ No newline at end of file
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/package-info.java
new file mode 100644
index 0000000..80d32bf
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.server;
+
+/**
+ * Server Util classes.
+ */
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 1fd54e5..a5d3c7d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -404,5 +404,11 @@ public final class OzoneConsts {
public static final String OM_RATIS_SNAPSHOT_DIR = "snapshot";
public static final String SCM_RATIS_SNAPSHOT_DIR = "snapshot";
- public static final long DEFAULT_OM_UPDATE_ID = -1L;
+ public static final long DEFAULT_OM_UPDATE_ID = -1L;
+
+
+ // SCM default service Id and node Id in non-HA where config is not defined
+ // in non-HA style.
+ public static final String SCM_DUMMY_NODEID = "scmNodeId";
+ public static final String SCM_DUMMY_SERVICE_ID = "scmServiceId";
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java
similarity index 100%
rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/package-info.java
new file mode 100644
index 0000000..5ed5162
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * Classes related to Ozone HA.
+ */
+package org.apache.hadoop.ozone.ha;
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index f27ad29..8fac16d 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1840,7 +1840,7 @@
</description>
</property>
<property>
- <name>ozone.scm.internal.service.id</name>
+ <name>ozone.scm.default.service.id</name>
<value></value>
<tag>OZONE, SCM, HA</tag>
<description>
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java
new file mode 100644
index 0000000..a88e1ca
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.ha;
+
+import org.apache.hadoop.hdds.conf.ConfigurationException;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY;
+
+public class TestSCMNodeInfo {
+
+ private OzoneConfiguration conf = new OzoneConfiguration();
+ private String scmServiceId = "scmserviceId";
+ private String[] nodes = new String[]{"scm1", "scm2", "scm3"};
+
+ @Before
+ public void setup() {
+ conf.set(ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY, scmServiceId);
+ conf.set(ScmConfigKeys.OZONE_SCM_NODES_KEY + "." + scmServiceId,
+ "scm1,scm2,scm3");
+ }
+
+ @Test
+ public void testScmHANodeInfo() {
+ int port = 9880;
+ for (String nodeId : nodes) {
+ conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY,
+ scmServiceId, nodeId), "localhost");
+ conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+ scmServiceId, nodeId), "localhost:" + ++port);
+ conf.setInt(ConfUtils.addKeySuffixes(OZONE_SCM_BLOCK_CLIENT_PORT_KEY,
+ scmServiceId, nodeId), port);
+
+ conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY,
+ scmServiceId, nodeId), "localhost:" + ++port);
+ conf.setInt(ConfUtils.addKeySuffixes(OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+ scmServiceId, nodeId), port);
+
+ conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_ADDRESS_KEY,
+ scmServiceId, nodeId), "localhost:" + ++port);
+ conf.setInt(ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_PORT_KEY,
+ scmServiceId, nodeId), port);
+
+ conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_DATANODE_ADDRESS_KEY,
+ scmServiceId, nodeId), "localhost:" + ++port);
+ conf.setInt(ConfUtils.addKeySuffixes(OZONE_SCM_DATANODE_PORT_KEY,
+ scmServiceId, nodeId), port);
+
+ }
+
+ List<SCMNodeInfo> scmNodeInfos = SCMNodeInfo.buildNodeInfo(conf);
+
+ port = 9880;
+
+ int count = 1;
+ for (SCMNodeInfo scmNodeInfo : scmNodeInfos) {
+ Assert.assertEquals(scmServiceId, scmNodeInfo.getServiceId());
+ Assert.assertEquals("scm"+count++, scmNodeInfo.getNodeId());
+ Assert.assertEquals("localhost:" + ++port,
+ scmNodeInfo.getBlockClientAddress());
+ Assert.assertEquals("localhost:" + ++port,
+ scmNodeInfo.getScmSecurityAddress());
+ Assert.assertEquals("localhost:" + ++port,
+ scmNodeInfo.getScmClientAddress());
+ Assert.assertEquals("localhost:" + ++port,
+ scmNodeInfo.getScmDatanodeAddress());
+ }
+ }
+
+ @Test
+ public void testSCMHANodeInfoWithDefaultPorts() {
+ for (String nodeId : nodes) {
+ conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY,
+ scmServiceId, nodeId), "localhost");
+ }
+
+ List<SCMNodeInfo> scmNodeInfos = SCMNodeInfo.buildNodeInfo(conf);
+
+ int count = 1;
+ for (SCMNodeInfo scmNodeInfo : scmNodeInfos) {
+ Assert.assertEquals(scmServiceId, scmNodeInfo.getServiceId());
+ Assert.assertEquals("scm"+count++, scmNodeInfo.getNodeId());
+ Assert.assertEquals("localhost:" + OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT,
+ scmNodeInfo.getBlockClientAddress());
+ Assert.assertEquals("localhost:" +
+ OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT,
+ scmNodeInfo.getScmSecurityAddress());
+ Assert.assertEquals("localhost:" + OZONE_SCM_CLIENT_PORT_DEFAULT,
+ scmNodeInfo.getScmClientAddress());
+ Assert.assertEquals("localhost:" + OZONE_SCM_DATANODE_PORT_DEFAULT,
+ scmNodeInfo.getScmDatanodeAddress());
+ }
+
+
+ }
+
+ @Test(expected = ConfigurationException.class)
+ public void testSCMHANodeInfoWithMissingSCMAddress() {
+ conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY,
+ scmServiceId, "scm1"), "localhost");
+ conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY,
+ scmServiceId, "scm1"), "localhost");
+
+ SCMNodeInfo.buildNodeInfo(conf);
+ }
+
+ @Test
+ public void testNonHAWithRestDefaults() {
+ OzoneConfiguration config = new OzoneConfiguration();
+
+ config.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost");
+
+ List< SCMNodeInfo > scmNodeInfos = SCMNodeInfo.buildNodeInfo(config);
+
+ Assert.assertNotNull(scmNodeInfos);
+ Assert.assertTrue(scmNodeInfos.size() == 1);
+ Assert.assertEquals("localhost:" + OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT,
+ scmNodeInfos.get(0).getBlockClientAddress());
+ Assert.assertEquals("localhost:" + OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT,
+ scmNodeInfos.get(0).getScmSecurityAddress());
+ Assert.assertEquals("localhost:" + OZONE_SCM_CLIENT_PORT_DEFAULT,
+ scmNodeInfos.get(0).getScmClientAddress());
+ Assert.assertEquals("localhost:" + OZONE_SCM_DATANODE_PORT_DEFAULT,
+ scmNodeInfos.get(0).getScmDatanodeAddress());
+ }
+
+
+}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/ha/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/ha/package-info.java
new file mode 100644
index 0000000..33ee79e
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/ha/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ Test cases for SCM HA Config loading.
+ */
+package org.apache.hadoop.hdds.scm.ha;
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java
index bcc1a01..eb90b5a 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java
@@ -20,8 +20,9 @@ package org.apache.hadoop.hdds.scm.proxy;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.io.retry.FailoverProxyProvider;
@@ -38,17 +39,11 @@ import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY;
-import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
-import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
-import static org.apache.hadoop.hdds.HddsUtils.getHostName;
+import static org.apache.hadoop.ozone.OzoneConsts.SCM_DUMMY_SERVICE_ID;
/**
* Failover proxy provider for SCM block location.
@@ -60,7 +55,7 @@ public class SCMBlockLocationFailoverProxyProvider implements
private Map<String, ProxyInfo<ScmBlockLocationProtocolPB>> scmProxies;
private Map<String, SCMProxyInfo> scmProxyInfoMap;
- private List<String> scmNodeIDList;
+ private List<String> scmNodeIds;
private String currentProxySCMNodeId;
private int currentProxyIndex;
@@ -68,82 +63,59 @@ public class SCMBlockLocationFailoverProxyProvider implements
private final ConfigurationSource conf;
private final long scmVersion;
- private final String scmServiceId;
+ private String scmServiceId;
private String lastAttemptedLeader;
private final int maxRetryCount;
private final long retryInterval;
- public static final String SCM_DUMMY_NODEID_PREFIX = "scm";
public SCMBlockLocationFailoverProxyProvider(ConfigurationSource conf) {
this.conf = conf;
this.scmVersion = RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
- this.scmServiceId = conf.getTrimmed(OZONE_SCM_SERVICE_IDS_KEY);
+
+ // Set some constant for non-HA.
+ if (scmServiceId == null) {
+ scmServiceId = SCM_DUMMY_SERVICE_ID;
+ }
this.scmProxies = new HashMap<>();
this.scmProxyInfoMap = new HashMap<>();
- this.scmNodeIDList = new ArrayList<>();
- loadConfigs();
+ loadConfigs();
this.currentProxyIndex = 0;
- currentProxySCMNodeId = scmNodeIDList.get(currentProxyIndex);
+ currentProxySCMNodeId = scmNodeIds.get(currentProxyIndex);
SCMClientConfig config = conf.getObject(SCMClientConfig.class);
this.maxRetryCount = config.getRetryCount();
this.retryInterval = config.getRetryInterval();
}
- @VisibleForTesting
- protected Collection<InetSocketAddress> getSCMAddressList() {
- Collection<String> scmAddressList =
- conf.getTrimmedStringCollection(OZONE_SCM_NAMES);
- Collection<InetSocketAddress> resultList = new ArrayList<>();
- if (!scmAddressList.isEmpty()) {
- final int port = getPortNumberFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY)
- .orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT);
- for (String scmAddress : scmAddressList) {
- LOG.info("SCM Address for proxy is {}", scmAddress);
-
- Optional<String> hostname = getHostName(scmAddress);
- if (hostname.isPresent()) {
- resultList.add(NetUtils.createSocketAddr(
- hostname.get() + ":" + port));
- }
- }
- }
- if (resultList.isEmpty()) {
- // fall back
- resultList.add(getScmAddressForBlockClients(conf));
- }
- return resultList;
- }
-
private void loadConfigs() {
- Collection<InetSocketAddress> scmAddressList = getSCMAddressList();
- int scmNodeIndex = 1;
- for (InetSocketAddress scmAddress : scmAddressList) {
- String nodeId = SCM_DUMMY_NODEID_PREFIX + scmNodeIndex;
- if (scmAddress == null) {
- LOG.error("Failed to create SCM proxy for {}.", nodeId);
- continue;
- }
- scmNodeIndex++;
- SCMProxyInfo scmProxyInfo = new SCMProxyInfo(
- scmServiceId, nodeId, scmAddress);
- ProxyInfo<ScmBlockLocationProtocolPB> proxy = new ProxyInfo<>(
- null, scmProxyInfo.toString());
- scmProxies.put(nodeId, proxy);
- scmProxyInfoMap.put(nodeId, scmProxyInfo);
- scmNodeIDList.add(nodeId);
- }
- if (scmProxies.isEmpty()) {
- throw new IllegalArgumentException("Could not find any configured " +
- "addresses for SCM. Please configure the system with "
- + OZONE_SCM_NAMES);
+ scmNodeIds = new ArrayList<>();
+ List<SCMNodeInfo> scmNodeInfoList = SCMNodeInfo.buildNodeInfo(conf);
+
+ for (SCMNodeInfo scmNodeInfo : scmNodeInfoList) {
+ if (scmNodeInfo.getBlockClientAddress() == null) {
+ throw new ConfigurationException("SCM BlockClient Address could not " +
+ "be obtained from config. Config is not properly defined");
+ } else {
+ InetSocketAddress scmBlockClientAddress =
+ NetUtils.createSocketAddr(scmNodeInfo.getBlockClientAddress());
+
+ scmServiceId = scmNodeInfo.getServiceId();
+ String scmNodeId = scmNodeInfo.getNodeId();
+ scmNodeIds.add(scmNodeId);
+ SCMProxyInfo scmProxyInfo = new SCMProxyInfo(
+ scmNodeInfo.getServiceId(), scmNodeInfo.getNodeId(),
+ scmBlockClientAddress);
+ ProxyInfo<ScmBlockLocationProtocolPB> proxy
+ = new ProxyInfo<>(null, scmProxyInfo.toString());
+ scmProxies.put(scmNodeId, proxy);
+ scmProxyInfoMap.put(scmNodeId, scmProxyInfo);
+ }
}
}
@@ -211,7 +183,7 @@ public class SCMBlockLocationFailoverProxyProvider implements
// round robin the next proxy
currentProxyIndex = (currentProxyIndex + 1) % scmProxies.size();
- currentProxySCMNodeId = scmNodeIDList.get(currentProxyIndex);
+ currentProxySCMNodeId = scmNodeIds.get(currentProxyIndex);
return currentProxyIndex;
}
@@ -220,7 +192,7 @@ public class SCMBlockLocationFailoverProxyProvider implements
if (scmProxies.containsKey(newLeaderNodeId)) {
lastAttemptedLeader = currentProxySCMNodeId;
currentProxySCMNodeId = newLeaderNodeId;
- currentProxyIndex = scmNodeIDList.indexOf(currentProxySCMNodeId);
+ currentProxyIndex = scmNodeIds.indexOf(currentProxySCMNodeId);
return true;
}
} else {
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java
index 03d084e..d501ba4 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java
@@ -20,8 +20,9 @@ package org.apache.hadoop.hdds.scm.proxy;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.io.retry.FailoverProxyProvider;
@@ -37,18 +38,9 @@ import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
-
-import static org.apache.hadoop.hdds.HddsUtils.getHostName;
-import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
-import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY;
/**
* Failover proxy provider for SCM container location.
@@ -60,7 +52,7 @@ public class SCMContainerLocationFailoverProxyProvider implements
private Map<String, ProxyInfo<StorageContainerLocationProtocolPB>> scmProxies;
private Map<String, SCMProxyInfo> scmProxyInfoMap;
- private List<String> scmNodeIDList;
+ private List<String> scmNodeIds;
private String currentProxySCMNodeId;
private int currentProxyIndex;
@@ -69,91 +61,53 @@ public class SCMContainerLocationFailoverProxyProvider implements
private final SCMClientConfig scmClientConfig;
private final long scmVersion;
- private final String scmServiceId;
+ private String scmServiceId;
private final int maxRetryCount;
private final long retryInterval;
- public static final String SCM_DUMMY_NODEID_PREFIX = "scm";
public SCMContainerLocationFailoverProxyProvider(ConfigurationSource conf) {
this.conf = conf;
this.scmVersion = RPC.getProtocolVersion(
StorageContainerLocationProtocolPB.class);
- this.scmServiceId = conf.getTrimmed(OZONE_SCM_SERVICE_IDS_KEY);
+
this.scmProxies = new HashMap<>();
this.scmProxyInfoMap = new HashMap<>();
- this.scmNodeIDList = new ArrayList<>();
loadConfigs();
this.currentProxyIndex = 0;
- currentProxySCMNodeId = scmNodeIDList.get(currentProxyIndex);
+ currentProxySCMNodeId = scmNodeIds.get(currentProxyIndex);
scmClientConfig = conf.getObject(SCMClientConfig.class);
this.maxRetryCount = scmClientConfig.getRetryCount();
this.retryInterval = scmClientConfig.getRetryInterval();
}
@VisibleForTesting
- protected Collection<InetSocketAddress> getSCMAddressList() {
- final int port = getPortNumberFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY)
- .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT);
-
- // TODO: modify the logic after SCM HA Conf is ready.
- // Use ozone.scm.client.address, if not set, fallback to ozone.scm.names.
- Optional<String> host = getHostNameFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
- if (host.isPresent()) {
- return Collections.singletonList(
- NetUtils.createSocketAddr(host.get() + ":" + port));
- }
-
- // Fallback to ozone.scm.names
- Collection<String> scmAddressList =
- conf.getTrimmedStringCollection(OZONE_SCM_NAMES);
-
- if (scmAddressList.isEmpty()) {
- throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_NAMES
- + " need to be a set of valid DNS names or IP addresses."
- + " Empty address list found.");
- }
-
- Collection<InetSocketAddress> resultList = new ArrayList<>();
-
- for (String scmAddress : scmAddressList) {
- LOG.debug("SCM Address for proxy is {}", scmAddress);
- Optional<String> hostname = getHostName(scmAddress);
- hostname.ifPresent(
- s -> resultList.add(NetUtils.createSocketAddr(s + ":" + port)));
- }
-
- return resultList;
- }
-
- private void loadConfigs() {
- Collection<InetSocketAddress> scmAddressList = getSCMAddressList();
- int scmNodeIndex = 1;
- for (InetSocketAddress scmAddress : scmAddressList) {
- String nodeId = SCM_DUMMY_NODEID_PREFIX + scmNodeIndex;
- if (scmAddress == null) {
- LOG.error("Failed to create SCM proxy for {}.", nodeId);
- continue;
+ protected void loadConfigs() {
+ List<SCMNodeInfo> scmNodeInfoList = SCMNodeInfo.buildNodeInfo(conf);
+
+ scmNodeIds = new ArrayList<>();
+
+ for (SCMNodeInfo scmNodeInfo : scmNodeInfoList) {
+ if (scmNodeInfo.getScmClientAddress() == null) {
+ throw new ConfigurationException("SCM Client Address could not " +
+ "be obtained from config. Config is not properly defined");
+ } else {
+ InetSocketAddress scmClientAddress =
+ NetUtils.createSocketAddr(scmNodeInfo.getScmClientAddress());
+
+ scmServiceId = scmNodeInfo.getServiceId();
+ String scmNodeId = scmNodeInfo.getNodeId();
+
+ scmNodeIds.add(scmNodeId);
+ SCMProxyInfo scmProxyInfo = new SCMProxyInfo(scmServiceId, scmNodeId,
+ scmClientAddress);
+ ProxyInfo< StorageContainerLocationProtocolPB > proxy
+ = new ProxyInfo<>(null, scmProxyInfo.toString());
+ scmProxies.put(scmNodeId, proxy);
+ scmProxyInfoMap.put(scmNodeId, scmProxyInfo);
}
- scmNodeIndex++;
- SCMProxyInfo scmProxyInfo = new SCMProxyInfo(
- scmServiceId, nodeId, scmAddress);
- ProxyInfo<StorageContainerLocationProtocolPB> proxy
- = new ProxyInfo<>(null, scmProxyInfo.toString());
- scmProxies.put(nodeId, proxy);
- scmProxyInfoMap.put(nodeId, scmProxyInfo);
- scmNodeIDList.add(nodeId);
- }
-
- if (scmProxies.isEmpty()) {
- throw new IllegalArgumentException("Could not find any configured " +
- "addresses for SCM. Please configure the system with "
- + OZONE_SCM_NAMES);
}
}
@@ -226,7 +180,7 @@ public class SCMContainerLocationFailoverProxyProvider implements
// round robin the next proxy
currentProxyIndex = (currentProxyIndex + 1) % scmProxies.size();
- currentProxySCMNodeId = scmNodeIDList.get(currentProxyIndex);
+ currentProxySCMNodeId = scmNodeIds.get(currentProxyIndex);
return currentProxyIndex;
}
@@ -235,7 +189,7 @@ public class SCMContainerLocationFailoverProxyProvider implements
if (scmProxies.containsKey(newLeaderNodeId)) {
// lastAttemptedLeader = currentProxySCMNodeId;
currentProxySCMNodeId = newLeaderNodeId;
- currentProxyIndex = scmNodeIDList.indexOf(currentProxySCMNodeId);
+ currentProxyIndex = scmNodeIds.indexOf(currentProxySCMNodeId);
return true;
}
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
index 6188262..ed5631b 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
@@ -157,7 +157,8 @@ public final class HddsServerUtil {
final int port = getPortNumberFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY)
- .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT);
+ .orElse(conf.getInt(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY,
+ ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
return NetUtils.createSocketAddr(host + ":" + port);
}
@@ -177,7 +178,8 @@ public final class HddsServerUtil {
final int port = getPortNumberFromConfigKeys(conf,
ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY)
- .orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT);
+ .orElse(conf.getInt(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_KEY,
+ ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
return NetUtils.createSocketAddr(host + ":" + port);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
index 6f36910..92a34f0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
@@ -31,12 +31,24 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.InetSocketAddress;
-import java.util.Collection;
import java.util.Optional;
import java.util.OptionalInt;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
/**
* SCM utility class.
@@ -72,86 +84,77 @@ public final class ScmUtils {
return dirFile;
}
- public static Collection<String> getSCMNodeIds(ConfigurationSource conf,
- String scmServiceId) {
- String key = ConfUtils.addSuffix(
- ScmConfigKeys.OZONE_SCM_NODES_KEY, scmServiceId);
- return conf.getTrimmedStringCollection(key);
- }
-
public static InetSocketAddress getScmBlockProtocolServerAddress(
OzoneConfiguration conf, String localScmServiceId, String nodeId) {
String bindHostKey = ConfUtils.addKeySuffixes(
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY,
- localScmServiceId, nodeId);
+ OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY, localScmServiceId, nodeId);
final Optional<String> host = getHostNameFromConfigKeys(conf, bindHostKey);
String addressKey = ConfUtils.addKeySuffixes(
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
- localScmServiceId, nodeId);
+ OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, localScmServiceId, nodeId);
final OptionalInt port = getPortNumberFromConfigKeys(conf, addressKey);
return NetUtils.createSocketAddr(
host.orElse(
ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT) + ":" +
- port.orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
+ port.orElse(conf.getInt(ConfUtils.addKeySuffixes(
+ OZONE_SCM_BLOCK_CLIENT_PORT_KEY, localScmServiceId, nodeId),
+ OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT)));
}
public static String getScmBlockProtocolServerAddressKey(
String serviceId, String nodeId) {
- return ConfUtils.addKeySuffixes(
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+ return ConfUtils.addKeySuffixes(OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
serviceId, nodeId);
}
public static InetSocketAddress getClientProtocolServerAddress(
OzoneConfiguration conf, String localScmServiceId, String nodeId) {
String bindHostKey = ConfUtils.addKeySuffixes(
- ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY,
- localScmServiceId, nodeId);
+ OZONE_SCM_CLIENT_BIND_HOST_KEY, localScmServiceId, nodeId);
final String host = getHostNameFromConfigKeys(conf, bindHostKey)
- .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT);
+ .orElse(OZONE_SCM_CLIENT_BIND_HOST_DEFAULT);
String addressKey = ConfUtils.addKeySuffixes(
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY,
- localScmServiceId, nodeId);
+ OZONE_SCM_CLIENT_ADDRESS_KEY, localScmServiceId, nodeId);
final int port = getPortNumberFromConfigKeys(conf, addressKey)
- .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT);
+ .orElse(conf.getInt(ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_PORT_KEY,
+ localScmServiceId, nodeId), OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
return NetUtils.createSocketAddr(host + ":" + port);
}
public static String getClientProtocolServerAddressKey(
String serviceId, String nodeId) {
- return ConfUtils.addKeySuffixes(
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY,
- serviceId, nodeId);
+ return ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_ADDRESS_KEY, serviceId,
+ nodeId);
}
public static InetSocketAddress getScmDataNodeBindAddress(
ConfigurationSource conf, String localScmServiceId, String nodeId) {
String bindHostKey = ConfUtils.addKeySuffixes(
- ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY,
+ OZONE_SCM_DATANODE_BIND_HOST_KEY,
localScmServiceId, nodeId
);
final Optional<String> host = getHostNameFromConfigKeys(conf, bindHostKey);
String addressKey = ConfUtils.addKeySuffixes(
- ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY,
- localScmServiceId, nodeId
- );
+ OZONE_SCM_DATANODE_ADDRESS_KEY, localScmServiceId,
+ nodeId);
final OptionalInt port = getPortNumberFromConfigKeys(conf, addressKey);
return NetUtils.createSocketAddr(
- host.orElse(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" +
- port.orElse(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
+ host.orElse(OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" +
+ port.orElse(conf.getInt(ConfUtils.addKeySuffixes(
+ OZONE_SCM_DATANODE_PORT_KEY, localScmServiceId, nodeId),
+ OZONE_SCM_DATANODE_PORT_DEFAULT)));
}
public static String getScmDataNodeBindAddressKey(
String serviceId, String nodeId) {
return ConfUtils.addKeySuffixes(
- ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY,
+ OZONE_SCM_DATANODE_ADDRESS_KEY,
serviceId, nodeId);
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
index 3edc7b3..499f2d5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
@@ -51,7 +51,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_K
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTPS_BIND_HOST_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTP_BIND_HOST_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_INTERNAL_SERVICE_ID;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PORT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PORT_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY;
@@ -137,7 +137,7 @@ public class SCMHANodeDetails {
Collection<String> scmServiceIds;
localScmServiceId = conf.getTrimmed(
- ScmConfigKeys.OZONE_SCM_INTERNAL_SERVICE_ID);
+ ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID);
LOG.info("ServiceID for StorageContainerManager is {}", localScmServiceId);
@@ -146,7 +146,7 @@ public class SCMHANodeDetails {
// .scm.service.ids.
LOG.info("{} is not defined, falling back to {} to find serviceID for "
+ "StorageContainerManager if it is HA enabled cluster",
- OZONE_SCM_INTERNAL_SERVICE_ID, OZONE_SCM_SERVICE_IDS_KEY);
+ OZONE_SCM_DEFAULT_SERVICE_ID, OZONE_SCM_SERVICE_IDS_KEY);
scmServiceIds = conf.getTrimmedStringCollection(
OZONE_SCM_SERVICE_IDS_KEY);
} else {
@@ -160,7 +160,7 @@ public class SCMHANodeDetails {
boolean isSCMddressSet = false;
for (String serviceId : scmServiceIds) {
- Collection<String> scmNodeIds = ScmUtils.getSCMNodeIds(conf, serviceId);
+ Collection<String> scmNodeIds = SCMHAUtils.getSCMNodeIds(conf, serviceId);
// TODO: need to fall back to ozone.scm.names in case scm node ids are
// not defined.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org