You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by GitBox <gi...@apache.org> on 2021/02/23 10:08:58 UTC
[GitHub] [ozone] bshashikant opened a new pull request #1953: Hdds 4718
bshashikant opened a new pull request #1953:
URL: https://github.com/apache/ozone/pull/1953
## What changes were proposed in this pull request?
Implemented sequence for adding new SCMs to existing cluster.
## What is the link to the Apache JIRA
https://issues.apache.org/jira/browse/HDDS-4718
## How was this patch tested?
Added UT
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583479003
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.Rule;
+
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
+
+/**
+ * Base class for Ozone Manager HA tests.
+ */
+public class TestStorageContainerManagerHA {
+
+ private MiniOzoneHAClusterImpl cluster = null;
+ private OzoneConfiguration conf;
+ private String clusterId;
+ private String scmId;
+ private String omServiceId;
+ private static int numOfOMs = 3;
+ private String scmServiceId;
+ // TODO: 3 servers will not work until SCM HA configs are considered in
+ // SCM HA Ratis server init
+ private static int numOfSCMs = 3;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omServiceId = "om-service-test1";
+ scmServiceId = "scm-service-test1";
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setScmId(scmId)
+ .setOMServiceId(omServiceId)
+ .setSCMServiceId(scmServiceId)
+ .setNumOfStorageContainerManagers(numOfSCMs)
+ .setNumOfOzoneManagers(numOfOMs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAllSCMAreRunning() throws Exception {
+ int count = 0;
+ List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
+ Assert.assertEquals(numOfSCMs, scms.size());
+ int peerSize = cluster.getStorageContainerManager().getScmHAManager()
+ .getRatisServer().getDivision().getGroup().getPeers().size();
+ for (StorageContainerManager scm : scms) {
+ if (scm.checkLeader()) {
+ count++;
+ }
+ Assert.assertTrue(peerSize == numOfSCMs);
+ }
+ Assert.assertEquals(1, count);
+ count = 0;
+ List<OzoneManager> oms = cluster.getOzoneManagersList();
+ Assert.assertEquals(numOfOMs, oms.size());
+ for (OzoneManager om : oms) {
+ if (om.isLeaderReady()) {
+ count++;
+ }
+ }
+ Assert.assertEquals(1, count);
+ testPutKey();
+ }
+
+ public void testPutKey() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ Instant testStartTime = Instant.now();
+ ObjectStore store =
+ OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length, STAND_ALONE, ONE,
+ new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
+ Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ is.close();
+ final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .setRefreshPipeline(true).build();
+ final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ final List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ final long containerID = keyLocationInfos.get(0).getContainerID();
+ for (int k = 0; k < numOfSCMs; k++) {
+ StorageContainerManager scm =
+ cluster.getStorageContainerManagers().get(k);
+ // flush to DB on each SCM
Review comment:
If 3rd SCM is not in the apply phase, takeSnapshot() will fail.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r582315387
##########
File path: hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
##########
@@ -311,8 +311,8 @@ public static InetSocketAddress getReconAddresses(
public static InetSocketAddress getSingleSCMAddress(
ConfigurationSource conf) {
Collection<InetSocketAddress> singleton = getSCMAddresses(conf);
- Preconditions.checkArgument(singleton.size() == 1,
- MULTIPLE_SCM_NOT_YET_SUPPORTED);
+ // Preconditions.checkArgument(singleton.size() == 1,
Review comment:
Thanks @vivekratnavel . We will remove this once the the proper SCM address validation patch is in . Its a work in progress.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bharatviswa504 commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bharatviswa504 commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583532028
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
+ TransactionInfo lastAppliedTrxInfo =
+ TransactionInfo.fromTermIndex(lastTermIndex);
+ if (transactionBuffer.getLatestTrxInfo().compareTo(lastAppliedTrxInfo)
+ < 0) {
+ transactionBuffer.updateLatestTrxInfo(
+ TransactionInfo.builder().setCurrentTerm(lastTermIndex.getTerm())
+ .setTransactionIndex(lastTermIndex.getIndex()).build());
+ transactionBuffer.setLatestSnapshot(
+ transactionBuffer.getLatestTrxInfo().toSnapshotInfo());
+ } else {
+ lastAppliedIndex =
+ transactionBuffer.getLatestTrxInfo().getTransactionIndex();
+ }
+
+ transactionBuffer.flush();
+ LOG.info("Current Snapshot Index {}, takeSnapshot took {} ms",
+ lastAppliedIndex, Time.monotonicNow() - startTime);
+ }
+ super.takeSnapshot();
+ return lastAppliedIndex;
+ }
+
+ @Override
+ public void initialize(
+ RaftServer server, RaftGroupId id, RaftStorage raftStorage)
+ throws IOException {
+ super.initialize(server, id, raftStorage);
+ storage.init(raftStorage);
+ loadSnapshot(storage.getLatestSnapshot());
+ }
+
+ private long loadSnapshot(SingleFileSnapshotInfo snapshot)
+ throws IOException {
+ if (snapshot == null) {
Review comment:
But this is not setting lastAppliedIndex, how before loadSnapshot used to set
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583451073
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
##########
@@ -326,6 +329,10 @@ private void initSCM() throws IOException {
scmStore.setScmId(scmId);
// writes the version file properties
scmStore.initialize();
+ if (!SCMHAUtils.isSCMHAEnabled(conf)) {
Review comment:
This step is required bcoz, otherwise, scm start won't elect a ratis leader as this is initializing scm version file without calling explicit scmInit.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583479527
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.Rule;
+
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
+
+/**
+ * Base class for Ozone Manager HA tests.
+ */
+public class TestStorageContainerManagerHA {
+
+ private MiniOzoneHAClusterImpl cluster = null;
+ private OzoneConfiguration conf;
+ private String clusterId;
+ private String scmId;
+ private String omServiceId;
+ private static int numOfOMs = 3;
+ private String scmServiceId;
+ // TODO: 3 servers will not work until SCM HA configs are considered in
+ // SCM HA Ratis server init
+ private static int numOfSCMs = 3;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omServiceId = "om-service-test1";
+ scmServiceId = "scm-service-test1";
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setScmId(scmId)
+ .setOMServiceId(omServiceId)
+ .setSCMServiceId(scmServiceId)
+ .setNumOfStorageContainerManagers(numOfSCMs)
+ .setNumOfOzoneManagers(numOfOMs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAllSCMAreRunning() throws Exception {
+ int count = 0;
+ List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
+ Assert.assertEquals(numOfSCMs, scms.size());
+ int peerSize = cluster.getStorageContainerManager().getScmHAManager()
+ .getRatisServer().getDivision().getGroup().getPeers().size();
+ for (StorageContainerManager scm : scms) {
+ if (scm.checkLeader()) {
+ count++;
+ }
+ Assert.assertTrue(peerSize == numOfSCMs);
+ }
+ Assert.assertEquals(1, count);
+ count = 0;
+ List<OzoneManager> oms = cluster.getOzoneManagersList();
+ Assert.assertEquals(numOfOMs, oms.size());
+ for (OzoneManager om : oms) {
+ if (om.isLeaderReady()) {
+ count++;
+ }
+ }
+ Assert.assertEquals(1, count);
+ testPutKey();
+ }
+
+ public void testPutKey() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ Instant testStartTime = Instant.now();
+ ObjectStore store =
+ OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length, STAND_ALONE, ONE,
+ new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
+ Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ is.close();
+ final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .setRefreshPipeline(true).build();
+ final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ final List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ final long containerID = keyLocationInfos.get(0).getContainerID();
+ for (int k = 0; k < numOfSCMs; k++) {
+ StorageContainerManager scm =
+ cluster.getStorageContainerManagers().get(k);
+ // flush to DB on each SCM
Review comment:
> If 3rd SCM is not in the apply phase, takeSnapshot() will fail. Snapshot guarantees whatever existing the logs commit is applied before snapshot is taken.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bharatviswa504 commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bharatviswa504 commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r582742292
##########
File path: hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
##########
@@ -379,12 +379,18 @@
// SCM Ratis related
public static final String OZONE_SCM_HA_ENABLE_KEY
= "ozone.scm.ratis.enable";
+ // setting this value to true, as right now, this switch is not accounted
+ // for spinning up SCM instances with/without SCM HA.
Review comment:
The comment says it is set to true, code still has false.
##########
File path: hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
##########
@@ -131,4 +134,24 @@ public static String getScmServiceId(ConfigurationSource conf) {
}
return localScmServiceId;
}
+
+ public static OzoneConfiguration removeSelfId(
Review comment:
Nit: JavaDoc missing
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
##########
@@ -69,6 +74,24 @@ public SCMHAManagerImpl(final ConfigurationSource conf,
@Override
public void start() throws IOException {
ratisServer.start();
+ if (SCMHAUtils.isSCMHAEnabled(conf) && ratisServer.getDivision().getGroup()
Review comment:
During init this check is removed, here added and the default is still false.
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
+ server.close();
+ }
+
+ private static void waitforLeaderToBeReady(RaftServer server, long timeout,
+ RaftGroup group)
+ throws IOException {
+ boolean ready;
+ long st = Time.monotonicNow();
+ do {
+ ready = server.getDivision(group.getGroupId()).getInfo().isLeaderReady();
+ if (!ready) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } while (!ready && Time.monotonicNow() - st < timeout);
- this.division =
- server.getDivision(server.getGroups().iterator().next().getGroupId());
+ if (!ready) {
+ throw new IOException(String
+ .format("Ratis group %s is not ready in %d ms", group.getGroupId(),
+ timeout));
+ }
}
- public static RaftServer.Builder newRaftServer(final String clusterId,
- final String scmId, final SCMHANodeDetails haDetails,
- final ConfigurationSource conf) throws IOException {
- final String scmNodeId = haDetails.getLocalNodeDetails().getNodeId();
+ private static RaftServer.Builder newRaftServer(final String scmId,
+ final ConfigurationSource conf) {
final SCMHAConfiguration haConf = conf.getObject(SCMHAConfiguration.class);
- SCMHAGroupBuilder haGrpBuilder = scmNodeId != null ?
- new SCMHAGroupBuilder(haDetails, clusterId, scmId) :
- new SCMHAGroupBuilder(haConf, conf, clusterId);
final RaftProperties serverProperties =
RatisUtil.newRaftProperties(haConf, conf);
- return RaftServer.newBuilder().setServerId(haGrpBuilder.getPeerId())
- .setGroup(haGrpBuilder.getRaftGroup()).setProperties(serverProperties)
- .setStateMachine(new SCMStateMachine());
+ return RaftServer.newBuilder().setServerId(RaftPeerId.getRaftPeerId(scmId))
+ .setProperties(serverProperties)
+ .setStateMachine(new SCMStateMachine(false));
}
@Override
public void start() throws IOException {
- division.getRaftServer().start();
+ server.start();
+ }
+
+ public RaftServer.Division getDivision() {
+ try {
+ return server
Review comment:
When failed throw exception instead of returning null?
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.Rule;
+
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
+
+/**
+ * Base class for Ozone Manager HA tests.
+ */
+public class TestStorageContainerManagerHA {
+
+ private MiniOzoneHAClusterImpl cluster = null;
+ private OzoneConfiguration conf;
+ private String clusterId;
+ private String scmId;
+ private String omServiceId;
+ private static int numOfOMs = 3;
+ private String scmServiceId;
+ // TODO: 3 servers will not work until SCM HA configs are considered in
+ // SCM HA Ratis server init
+ private static int numOfSCMs = 3;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omServiceId = "om-service-test1";
+ scmServiceId = "scm-service-test1";
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setScmId(scmId)
+ .setOMServiceId(omServiceId)
+ .setSCMServiceId(scmServiceId)
+ .setNumOfStorageContainerManagers(numOfSCMs)
+ .setNumOfOzoneManagers(numOfOMs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAllSCMAreRunning() throws Exception {
+ int count = 0;
+ List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
+ Assert.assertEquals(numOfSCMs, scms.size());
+ int peerSize = cluster.getStorageContainerManager().getScmHAManager()
+ .getRatisServer().getDivision().getGroup().getPeers().size();
+ for (StorageContainerManager scm : scms) {
+ if (scm.checkLeader()) {
+ count++;
+ }
+ Assert.assertTrue(peerSize == numOfSCMs);
+ }
+ Assert.assertEquals(1, count);
+ count = 0;
+ List<OzoneManager> oms = cluster.getOzoneManagersList();
+ Assert.assertEquals(numOfOMs, oms.size());
+ for (OzoneManager om : oms) {
+ if (om.isLeaderReady()) {
+ count++;
+ }
+ }
+ Assert.assertEquals(1, count);
+ testPutKey();
+ }
+
+ public void testPutKey() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ Instant testStartTime = Instant.now();
+ ObjectStore store =
+ OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length, STAND_ALONE, ONE,
+ new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
+ Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ is.close();
+ final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .setRefreshPipeline(true).build();
+ final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ final List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ final long containerID = keyLocationInfos.get(0).getContainerID();
+ for (int k = 0; k < numOfSCMs; k++) {
+ StorageContainerManager scm =
+ cluster.getStorageContainerManagers().get(k);
+ // flush to DB on each SCM
Review comment:
Can we check if with out flush also, it is available on all SCMS.
And also for this test, do you think we need to make sure first that each SCM catch up with transaction and then do checks. (As when request is processed, only leader applies to StateMachine)
##########
File path: hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
##########
@@ -53,6 +55,23 @@ public static ScmInfo getScmInfo(OzoneConfiguration conf)
}
}
+ public static boolean addSCM(OzoneConfiguration conf, AddSCMRequest request)
+ throws IOException {
+ OzoneConfiguration config = SCMHAUtils.removeSelfId(conf);
+ try {
+ RetryPolicy retryPolicy = retryUpToMaximumCountWithFixedSleep(
+ 10, 5, TimeUnit.SECONDS);
+ RetriableTask<Boolean> retriable = new RetriableTask<>(
+ retryPolicy, "addSCM",
+ () -> getScmBlockClient(config).addSCM(request));
+ return retriable.call();
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException("Failed to get SCM info", e);
Review comment:
Minor: Failed to add SCM
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
Review comment:
60000 hardcoded value, can we make this config?
##########
File path: hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
##########
@@ -22,14 +22,17 @@
import com.google.common.base.Strings;
import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.ozone.ha.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.nio.file.Paths;
-import java.util.Collection;
+import java.util.*;
Review comment:
Minor: Expand * import.
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
##########
@@ -291,6 +292,35 @@ public ScmInfo getScmInfo() throws IOException {
}
}
+ @Override
+ public boolean addSCM(AddSCMRequest request) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding SCM {} addr {} with cluster id {}, with {}",
Review comment:
Only 3 parameters, we have 4 {}
##########
File path: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
##########
@@ -765,12 +765,18 @@ public void addGroup(HddsProtos.PipelineID pipelineId,
clientId, server.getId(), nextCallId(), group);
RaftClientReply reply;
+ if (LOG.isDebugEnabled()) {
Review comment:
We donot require isDebugEnabled
http://www.slf4j.org/faq.html#logging_performance
```
Better yet, use parameterized messages
There exists a very convenient alternative based on message formats. Assuming entry is an object, you can write:
Object entry = new SomeObject();
logger.debug("The entry is {}.", entry);
After evaluating whether to log or not, and only if the decision is affirmative, will the logger implementation format the message and replace the '{}' pair with the string value of entry. In other words, this form does not incur the cost of parameter construction in case the log statement is disabled.
```
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
##########
@@ -291,6 +292,35 @@ public ScmInfo getScmInfo() throws IOException {
}
}
+ @Override
+ public boolean addSCM(AddSCMRequest request) throws IOException {
+ if (LOG.isDebugEnabled()) {
Review comment:
Minor: We don't need isDebugEnabled()
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
##########
@@ -1169,11 +1168,12 @@ public ReplicationManager getReplicationManager() {
}
/**
- * Check if the current scm is the leader.
- * @return - if the current scm is the leader.
+ * Check if the current scm is the leader and ready for accepting requests.
+ * @return - if the current scm is the leader and is ready.
*/
public boolean checkLeader() {
- return scmContext.isLeader();
+ return scmContext.isLeader() && getScmHAManager().getRatisServer()
Review comment:
Question Do we need this && check here. Ratis is source of truth, so getScmHAManager().getRatisServer().getDivision().getInfo().isLeaderReady() will suffice here?
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -156,16 +200,52 @@ public void stop() throws IOException {
@Override
public NotLeaderException triggerNotLeaderException() {
return new NotLeaderException(
- division.getMemberId(), null, division.getGroup().getPeers());
+ getDivision().getMemberId(), null, getDivision().getGroup().getPeers());
+ }
+
+ @Override
+ public boolean addSCM(AddSCMRequest request) throws IOException {
+ List<RaftPeer> newRaftPeerList =
+ new ArrayList<>(getDivision().getGroup().getPeers());
+ // add the SCM node to be added to the raft peer list
+
+ RaftPeer raftPeer = RaftPeer.newBuilder().setId(request.getScmId())
+ .setAddress(request.getRatisAddr()).build();
+ newRaftPeerList.add(raftPeer);
+
+ LOG.debug("{}: Submitting SetConfiguration request to Ratis server with" +
Review comment:
Can we make this a info log
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
##########
@@ -708,14 +708,14 @@ public static boolean scmBootstrap(OzoneConfiguration conf)
}
// The node here will try to fetch the cluster id from any of existing
// running SCM instances.
- // TODO: need to avoid failover to local SCM Node here
- final ScmInfo scmInfo = HAUtils.getScmInfo(conf);
Review comment:
Here we need to call SCMHANodeDetails.loadSCMHAConfig(conf);
Which will set node specific config key and then call getLocalNodeDetails to find localNodeId.
Then this localNodeId can be removed.
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
##########
@@ -689,9 +689,9 @@ public static String buildRpcServerStartMessage(String description,
}
/**
- * Routine to bootstrap the StorageContainerManager. Thsi will connect to a
+ * Routine to bootstrap the StorageContainerManager. Thiw will connect to a
Review comment:
Minor: Thiw -> This
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
##########
@@ -291,6 +292,35 @@ public ScmInfo getScmInfo() throws IOException {
}
}
+ @Override
+ public boolean addSCM(AddSCMRequest request) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding SCM {} addr {} with cluster id {}, with {}",
+ request.getScmId(), request.getRatisAddr(), request.getRatisAddr());
+ }
+
+ Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("scmId", String.valueOf(request.getScmId()));
+ auditMap.put("cluster", String.valueOf(request.getClusterId()));
+ auditMap.put("addr", String.valueOf(request.getRatisAddr()));
+ boolean auditSuccess = true;
+ try{
+ return scm.getScmHAManager().addSCM(request);
+ } catch (Exception ex) {
+ auditSuccess = false;
+ AUDIT.logReadFailure(
+ buildAuditMessageForFailure(SCMAction.ADD_SCM, auditMap, ex)
+ );
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logReadSuccess(
+ buildAuditMessageForSuccess(SCMAction.ADD_SCM, null)
Review comment:
null -> auditMap
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
+ server.close();
+ }
+
+ private static void waitforLeaderToBeReady(RaftServer server, long timeout,
+ RaftGroup group)
+ throws IOException {
+ boolean ready;
+ long st = Time.monotonicNow();
+ do {
+ ready = server.getDivision(group.getGroupId()).getInfo().isLeaderReady();
+ if (!ready) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } while (!ready && Time.monotonicNow() - st < timeout);
- this.division =
- server.getDivision(server.getGroups().iterator().next().getGroupId());
+ if (!ready) {
+ throw new IOException(String
+ .format("Ratis group %s is not ready in %d ms", group.getGroupId(),
+ timeout));
+ }
}
- public static RaftServer.Builder newRaftServer(final String clusterId,
- final String scmId, final SCMHANodeDetails haDetails,
- final ConfigurationSource conf) throws IOException {
- final String scmNodeId = haDetails.getLocalNodeDetails().getNodeId();
+ private static RaftServer.Builder newRaftServer(final String scmId,
+ final ConfigurationSource conf) {
final SCMHAConfiguration haConf = conf.getObject(SCMHAConfiguration.class);
- SCMHAGroupBuilder haGrpBuilder = scmNodeId != null ?
- new SCMHAGroupBuilder(haDetails, clusterId, scmId) :
- new SCMHAGroupBuilder(haConf, conf, clusterId);
final RaftProperties serverProperties =
RatisUtil.newRaftProperties(haConf, conf);
- return RaftServer.newBuilder().setServerId(haGrpBuilder.getPeerId())
- .setGroup(haGrpBuilder.getRaftGroup()).setProperties(serverProperties)
- .setStateMachine(new SCMStateMachine());
+ return RaftServer.newBuilder().setServerId(RaftPeerId.getRaftPeerId(scmId))
+ .setProperties(serverProperties)
+ .setStateMachine(new SCMStateMachine(false));
}
@Override
public void start() throws IOException {
- division.getRaftServer().start();
+ server.start();
+ }
+
+ public RaftServer.Division getDivision() {
+ try {
+ return server
+ .getDivision(server.getGroups().iterator().next().getGroupId());
+ } catch (Exception e) {
+ LOG.warn("Failed to get RaftServerDivision", e);
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ public SCMStateMachine getStateMachine() {
+ return stateMachine;
}
@Override
public void registerStateMachineHandler(final RequestType handlerType,
final Object handler) {
- ((SCMStateMachine) division.getStateMachine())
- .registerHandler(handlerType, handler);
+ stateMachine.registerHandler(handlerType, handler);
}
@Override
public SCMRatisResponse submitRequest(SCMRatisRequest request)
throws IOException, ExecutionException, InterruptedException {
final RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder()
.setClientId(clientId)
Review comment:
Not related to this, but ClientID should be set with Server.getClientID() so that when requests retried, we know it is a retry request. With this after SCM restart, all old requests will come with new ClientID
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
+ server.close();
+ }
+
+ private static void waitforLeaderToBeReady(RaftServer server, long timeout,
+ RaftGroup group)
+ throws IOException {
+ boolean ready;
+ long st = Time.monotonicNow();
+ do {
+ ready = server.getDivision(group.getGroupId()).getInfo().isLeaderReady();
+ if (!ready) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } while (!ready && Time.monotonicNow() - st < timeout);
- this.division =
- server.getDivision(server.getGroups().iterator().next().getGroupId());
+ if (!ready) {
+ throw new IOException(String
+ .format("Ratis group %s is not ready in %d ms", group.getGroupId(),
+ timeout));
+ }
}
- public static RaftServer.Builder newRaftServer(final String clusterId,
- final String scmId, final SCMHANodeDetails haDetails,
- final ConfigurationSource conf) throws IOException {
- final String scmNodeId = haDetails.getLocalNodeDetails().getNodeId();
+ private static RaftServer.Builder newRaftServer(final String scmId,
+ final ConfigurationSource conf) {
final SCMHAConfiguration haConf = conf.getObject(SCMHAConfiguration.class);
- SCMHAGroupBuilder haGrpBuilder = scmNodeId != null ?
- new SCMHAGroupBuilder(haDetails, clusterId, scmId) :
- new SCMHAGroupBuilder(haConf, conf, clusterId);
final RaftProperties serverProperties =
RatisUtil.newRaftProperties(haConf, conf);
- return RaftServer.newBuilder().setServerId(haGrpBuilder.getPeerId())
- .setGroup(haGrpBuilder.getRaftGroup()).setProperties(serverProperties)
- .setStateMachine(new SCMStateMachine());
+ return RaftServer.newBuilder().setServerId(RaftPeerId.getRaftPeerId(scmId))
+ .setProperties(serverProperties)
+ .setStateMachine(new SCMStateMachine(false));
}
@Override
public void start() throws IOException {
- division.getRaftServer().start();
+ server.start();
+ }
+
+ public RaftServer.Division getDivision() {
+ try {
+ return server
+ .getDivision(server.getGroups().iterator().next().getGroupId());
+ } catch (Exception e) {
+ LOG.warn("Failed to get RaftServerDivision", e);
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ public SCMStateMachine getStateMachine() {
+ return stateMachine;
}
@Override
public void registerStateMachineHandler(final RequestType handlerType,
final Object handler) {
- ((SCMStateMachine) division.getStateMachine())
- .registerHandler(handlerType, handler);
+ stateMachine.registerHandler(handlerType, handler);
}
@Override
public SCMRatisResponse submitRequest(SCMRatisRequest request)
throws IOException, ExecutionException, InterruptedException {
final RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder()
.setClientId(clientId)
- .setServerId(division.getId())
- .setGroupId(division.getGroup().getGroupId())
+ .setServerId(getDivision().getId())
Review comment:
Can we store this locally instead of every time fetching from Ratis APIS?
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
+ TransactionInfo lastAppliedTrxInfo =
+ TransactionInfo.fromTermIndex(lastTermIndex);
+ if (transactionBuffer.getLatestTrxInfo().compareTo(lastAppliedTrxInfo)
+ < 0) {
+ transactionBuffer.updateLatestTrxInfo(
+ TransactionInfo.builder().setCurrentTerm(lastTermIndex.getTerm())
+ .setTransactionIndex(lastTermIndex.getIndex()).build());
+ transactionBuffer.setLatestSnapshot(
+ transactionBuffer.getLatestTrxInfo().toSnapshotInfo());
+ } else {
+ lastAppliedIndex =
+ transactionBuffer.getLatestTrxInfo().getTransactionIndex();
+ }
+
+ transactionBuffer.flush();
+ LOG.info("Current Snapshot Index {}, takeSnapshot took {} ms",
+ lastAppliedIndex, Time.monotonicNow() - startTime);
+ }
+ super.takeSnapshot();
Review comment:
Do we need this call?
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
##########
@@ -326,6 +329,10 @@ private void initSCM() throws IOException {
scmStore.setScmId(scmId);
// writes the version file properties
scmStore.initialize();
+ if (!SCMHAUtils.isSCMHAEnabled(conf)) {
Review comment:
Question: Why do we need this step here?
##########
File path: hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
##########
@@ -77,6 +78,11 @@
*/
ScmInfo getScmInfo() throws IOException;
+ /**
+ * Request to add SCM instance to HA group.
+ */
+ boolean addSCM(AddSCMRequest request) throws IOException;
Review comment:
I also believe this is not the correct place for addSCM, as StorageContainerLocation is used for all admin ops, that might be good place. We need to conclude moving this to StorageContainerLocationProtocol before merge.
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
+ TransactionInfo lastAppliedTrxInfo =
+ TransactionInfo.fromTermIndex(lastTermIndex);
+ if (transactionBuffer.getLatestTrxInfo().compareTo(lastAppliedTrxInfo)
+ < 0) {
+ transactionBuffer.updateLatestTrxInfo(
+ TransactionInfo.builder().setCurrentTerm(lastTermIndex.getTerm())
+ .setTransactionIndex(lastTermIndex.getIndex()).build());
+ transactionBuffer.setLatestSnapshot(
+ transactionBuffer.getLatestTrxInfo().toSnapshotInfo());
+ } else {
+ lastAppliedIndex =
+ transactionBuffer.getLatestTrxInfo().getTransactionIndex();
+ }
+
+ transactionBuffer.flush();
+ LOG.info("Current Snapshot Index {}, takeSnapshot took {} ms",
+ lastAppliedIndex, Time.monotonicNow() - startTime);
+ }
+ super.takeSnapshot();
+ return lastAppliedIndex;
+ }
+
+ @Override
+ public void initialize(
+ RaftServer server, RaftGroupId id, RaftStorage raftStorage)
+ throws IOException {
+ super.initialize(server, id, raftStorage);
+ storage.init(raftStorage);
+ loadSnapshot(storage.getLatestSnapshot());
+ }
+
+ private long loadSnapshot(SingleFileSnapshotInfo snapshot)
+ throws IOException {
+ if (snapshot == null) {
Review comment:
Here load snapshot should be loaded from TransactionInfo table?
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.Rule;
+
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
+
+/**
+ * Base class for Ozone Manager HA tests.
+ */
+public class TestStorageContainerManagerHA {
+
+ private MiniOzoneHAClusterImpl cluster = null;
+ private OzoneConfiguration conf;
+ private String clusterId;
+ private String scmId;
+ private String omServiceId;
+ private static int numOfOMs = 3;
+ private String scmServiceId;
+ // TODO: 3 servers will not work until SCM HA configs are considered in
+ // SCM HA Ratis server init
+ private static int numOfSCMs = 3;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omServiceId = "om-service-test1";
+ scmServiceId = "scm-service-test1";
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setScmId(scmId)
+ .setOMServiceId(omServiceId)
+ .setSCMServiceId(scmServiceId)
+ .setNumOfStorageContainerManagers(numOfSCMs)
+ .setNumOfOzoneManagers(numOfOMs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAllSCMAreRunning() throws Exception {
+ int count = 0;
+ List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
+ Assert.assertEquals(numOfSCMs, scms.size());
+ int peerSize = cluster.getStorageContainerManager().getScmHAManager()
+ .getRatisServer().getDivision().getGroup().getPeers().size();
+ for (StorageContainerManager scm : scms) {
+ if (scm.checkLeader()) {
+ count++;
+ }
+ Assert.assertTrue(peerSize == numOfSCMs);
+ }
+ Assert.assertEquals(1, count);
+ count = 0;
+ List<OzoneManager> oms = cluster.getOzoneManagersList();
+ Assert.assertEquals(numOfOMs, oms.size());
+ for (OzoneManager om : oms) {
+ if (om.isLeaderReady()) {
+ count++;
+ }
+ }
+ Assert.assertEquals(1, count);
+ testPutKey();
+ }
+
+ public void testPutKey() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ Instant testStartTime = Instant.now();
+ ObjectStore store =
+ OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length, STAND_ALONE, ONE,
+ new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
+ Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ is.close();
+ final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .setRefreshPipeline(true).build();
+ final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ final List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ final long containerID = keyLocationInfos.get(0).getContainerID();
+ for (int k = 0; k < numOfSCMs; k++) {
+ StorageContainerManager scm =
+ cluster.getStorageContainerManagers().get(k);
+ // flush to DB on each SCM
Review comment:
Question: Does without flush also container should be there right?
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
Review comment:
Here there is a chance of missing DB update.
Lets take 1,2,4 metadata/confUpdate and 3 is applyTransaction (Still not added to TransactionDBBuffer)
And now when **takeSnapshot** lastAppliedTermIndex will be 4, (As notifyConfigurationChanged will be called) and we flush. Restart SCM, we think 4 is applied but it is not.
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
Review comment:
We are waiting for leader to be ready because RaftGroup on disk will be created after leader election?
##########
File path: hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
##########
@@ -53,6 +55,23 @@ public static ScmInfo getScmInfo(OzoneConfiguration conf)
}
}
+ public static boolean addSCM(OzoneConfiguration conf, AddSCMRequest request)
+ throws IOException {
+ OzoneConfiguration config = SCMHAUtils.removeSelfId(conf);
+ try {
+ RetryPolicy retryPolicy = retryUpToMaximumCountWithFixedSleep(
+ 10, 5, TimeUnit.SECONDS);
+ RetriableTask<Boolean> retriable = new RetriableTask<>(
+ retryPolicy, "addSCM",
+ () -> getScmBlockClient(config).addSCM(request));
+ return retriable.call();
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException("Failed to get SCM info", e);
Review comment:
Question: Do we need this retry policy here?
As SCMFailOverProxy provider will have its own retry count and retry interval.
```
this.maxRetryCount = config.getRetryCount();
this.retryInterval = config.getRetryInterval();
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bharatviswa504 commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bharatviswa504 commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583473781
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
##########
@@ -0,0 +1,738 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.safemode.HealthyPipelineSafeModeRule;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.recon.ReconServer;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.BindException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * MiniOzoneHAClusterImpl creates a complete in-process Ozone cluster
+ * with OM HA and SCM HA suitable for running tests.
+ * The cluster consists of a set of
+ * OzoneManagers, StorageContainerManagers and multiple DataNodes.
+ */
+public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MiniOzoneHAClusterImpl.class);
+
+ private final OMHAService omhaService;
+ private final SCMHAService scmhaService;
+ private static ObjectStore store = null;
Review comment:
Minor:
private static ObjectStore store = null; Unused variable
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
##########
@@ -0,0 +1,738 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.safemode.HealthyPipelineSafeModeRule;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.recon.ReconServer;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.BindException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * MiniOzoneHAClusterImpl creates a complete in-process Ozone cluster
+ * with OM HA and SCM HA suitable for running tests.
+ * The cluster consists of a set of
+ * OzoneManagers, StorageContainerManagers and multiple DataNodes.
+ */
+public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MiniOzoneHAClusterImpl.class);
+
+ private final OMHAService omhaService;
+ private final SCMHAService scmhaService;
+ private static ObjectStore store = null;
+
+ private int waitForClusterToBeReadyTimeout = 120000; // 2 min
+
+ private static final Random RANDOM = new Random();
+ private static final int RATIS_RPC_TIMEOUT = 1000; // 1 second
+ public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
+
+ /**
+ * Creates a new MiniOzoneCluster.
+ *
+ * @throws IOException if there is an I/O error
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> activeOMList,
+ List<OzoneManager> inactiveOMList,
+ List<StorageContainerManager> activeSCMList,
+ List<StorageContainerManager> inactiveSCMList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId,
+ ReconServer reconServer) {
+ super(conf, hddsDatanodes, reconServer);
+ omhaService =
+ new OMHAService(activeOMList, inactiveOMList, omServiceId);
+ scmhaService =
+ new SCMHAService(activeSCMList, inactiveSCMList, scmServiceId);
+ }
+
+ /**
+ * Creates a new MiniOzoneCluster with all OMs active.
+ * This is used by MiniOzoneChaosCluster.
+ */
+ protected MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> omList,
+ List<StorageContainerManager> scmList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId) {
+ this(conf, omList, null, scmList, null, hddsDatanodes,
+ omServiceId, scmServiceId, null);
+ }
+
+ @Override
+ public String getOMServiceId() {
+ return omhaService.getServiceId();
+ }
+
+ @Override
+ public String getSCMServiceId() {
+ return scmhaService.getServiceId();
+ }
+
+ /**
+ * Returns the first OzoneManager from the list.
+ * @return
+ */
+ @Override
+ public OzoneManager getOzoneManager() {
+ return this.omhaService.getServices().get(0);
+ }
+
+ @Override
+ public OzoneClient getRpcClient() throws IOException {
+ String omServiceId = omhaService.getServiceId();
+ if (omServiceId == null) {
+ // Non-HA cluster.
+ return OzoneClientFactory.getRpcClient(getConf());
+ } else {
+ // HA cluster
+ return OzoneClientFactory.getRpcClient(omServiceId, getConf());
+ }
+ }
+
+ public boolean isOMActive(String omNodeId) {
+ return omhaService.isServiceActive(omNodeId);
+ }
+
+ public OzoneManager getOzoneManager(int index) {
+ return this.omhaService.getServiceByIndex(index);
+ }
+
+ public OzoneManager getOzoneManager(String omNodeId) {
+ return this.omhaService.getServiceById(omNodeId);
+ }
+
+ public List<OzoneManager> getOzoneManagersList() {
+ return omhaService.getServices();
+ }
+
+ /**
+ * Get OzoneManager leader object.
+ * @return OzoneManager object, null if there isn't one or more than one
+ */
+ public OzoneManager getOMLeader() {
+ OzoneManager res = null;
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager.isLeaderReady()) {
+ if (res != null) {
+ // Found more than one leader
+ // Return null, expect the caller to retry in a while
+ return null;
+ }
+ // Found a leader
+ res = ozoneManager;
+ }
+ }
+ return res;
+ }
+
+ /**
+ * Start a previously inactive OM.
+ */
+ public void startInactiveOM(String omNodeID) throws IOException {
+ omhaService.startInactiveService(omNodeID, OzoneManager::start);
+ }
+
+ @Override
+ public void restartOzoneManager() throws IOException {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ ozoneManager.stop();
+ ozoneManager.restart();
+ }
+ }
+
+ public void shutdownOzoneManager(OzoneManager ozoneManager) {
+ LOG.info("Shutting down OzoneManager " + ozoneManager.getOMNodeId());
+
+ ozoneManager.stop();
+ }
+
+ public void restartOzoneManager(OzoneManager ozoneManager, boolean waitForOM)
+ throws IOException, TimeoutException, InterruptedException {
+ LOG.info("Restarting OzoneManager " + ozoneManager.getOMNodeId());
+ ozoneManager.restart();
+
+ if (waitForOM) {
+ GenericTestUtils.waitFor(ozoneManager::isRunning,
+ 1000, waitForClusterToBeReadyTimeout);
+ }
+ }
+
+ public String getClusterId() throws IOException {
+ return scmhaService.getServices().get(0)
+ .getClientProtocolServer().getScmInfo().getClusterId();
+ }
+
+ public StorageContainerManager getActiveSCM() {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return scm;
+ }
+ }
+ return null;
+ }
+
+ public void waitForSCMToBeReady()
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(() -> {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return true;
+ }
+ }
+ return false;
+ }, 1000, waitForClusterToBeReadyTimeout);
+ }
+
+ @Override
+ public void stop() {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager != null) {
+ LOG.info("Stopping the OzoneManager {}", ozoneManager.getOMNodeId());
+ ozoneManager.stop();
+ ozoneManager.join();
+ }
+ }
+
+ for (StorageContainerManager scm : this.scmhaService.getServices()) {
+ if (scm != null) {
+ LOG.info("Stopping the StorageContainerManager {}", scm.getScmId());
+ scm.stop();
+ scm.join();
+ }
+ }
+ super.stop();
+ }
+
+ public void stopOzoneManager(int index) {
+ omhaService.getServices().get(index).stop();
+ omhaService.getServices().get(index).join();
+ }
+
+ public void stopOzoneManager(String omNodeId) {
+ omhaService.getServiceById(omNodeId).stop();
+ omhaService.getServiceById(omNodeId).join();
+ }
+
+ /**
+ * Builder for configuring the MiniOzoneCluster to run.
+ */
+ public static class Builder extends MiniOzoneClusterImpl.Builder {
+
+ private static final String OM_NODE_ID_PREFIX = "omNode-";
+ private List<OzoneManager> activeOMs = new ArrayList<>();
+ private List<OzoneManager> inactiveOMs = new ArrayList<>();
+
+ private static final String SCM_NODE_ID_PREFIX = "scmNode-";
+ private List<StorageContainerManager> activeSCMs = new ArrayList<>();
+ private List<StorageContainerManager> inactiveSCMs = new ArrayList<>();
+
+ /**
+ * Creates a new Builder.
+ *
+ * @param conf configuration
+ */
+ public Builder(OzoneConfiguration conf) {
+ super(conf);
+ }
+
+ public List<OzoneManager> getActiveOMs() {
+ return activeOMs;
+ }
+
+ public List<OzoneManager> getInactiveOMs() {
+ return inactiveOMs;
+ }
+
+ @Override
+ public MiniOzoneCluster build() throws IOException {
+ if (numOfActiveOMs > numOfOMs) {
+ throw new IllegalArgumentException("Number of active OMs cannot be " +
+ "more than the total number of OMs");
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveOMs == ACTIVE_OMS_NOT_SET) {
+ numOfActiveOMs = numOfOMs;
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveSCMs == ACTIVE_SCMS_NOT_SET) {
+ numOfActiveSCMs = numOfSCMs;
+ }
+
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ initializeConfiguration();
+ initOMRatisConf();
+ StorageContainerManager scm;
+ ReconServer reconServer = null;
+ try {
+ createSCMService();
+ createOMService();
+ if (includeRecon) {
+ configureRecon();
+ reconServer = new ReconServer();
+ reconServer.execute(new String[] {});
+ }
+ } catch (AuthenticationException ex) {
+ throw new IOException("Unable to build MiniOzoneCluster. ", ex);
+ }
+
+ final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(
+ activeSCMs, reconServer);
+
+ MiniOzoneHAClusterImpl cluster = new MiniOzoneHAClusterImpl(conf,
+ activeOMs, inactiveOMs, activeSCMs, inactiveSCMs,
+ hddsDatanodes, omServiceId, scmServiceId, reconServer);
+
+ if (startDataNodes) {
+ cluster.startHddsDatanodes();
+ }
+ return cluster;
+ }
+
+ protected void initOMRatisConf() {
+ conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
+ conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers);
+
+ // If test change the following config values we will respect,
+ // otherwise we will set lower timeout values.
+ long defaultDuration = OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT
+ .getDuration();
+ long curRatisRpcTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration == curRatisRpcTimeout ?
+ RATIS_RPC_TIMEOUT : curRatisRpcTimeout, TimeUnit.MILLISECONDS);
+
+ long defaultNodeFailureTimeout =
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getDuration();
+ long curNodeFailureTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ defaultNodeFailureTimeout,
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getUnit());
+ conf.setTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ curNodeFailureTimeout == defaultNodeFailureTimeout ?
+ NODE_FAILURE_TIMEOUT : curNodeFailureTimeout,
+ TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<OzoneManager> createOMService() throws IOException,
+ AuthenticationException {
+
+ List<OzoneManager> omList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 10000;
+
+ while (true) {
+ try {
+ basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ initOMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfOMs; i++) {
+ // Set nodeId
+ String nodeId = OM_NODE_ID_PREFIX + i;
+ OzoneConfiguration config = new OzoneConfiguration(conf);
+ config.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeId);
+ // Set the OM http(s) address to null so that the cluster picks
+ // up the address set with service ID and node ID in initHAConfig
+ config.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "");
+ config.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, "");
+
+ // Set metadata/DB dir base path
+ String metaDirPath = path + "/" + nodeId;
+ config.set(OZONE_METADATA_DIRS, metaDirPath);
+ // OMStorage omStore = new OMStorage(config);
+ // initializeOmStorage(omStore);
+ OzoneManager.omInit(config);
+ OzoneManager om = OzoneManager.createOm(config);
+ if (certClient != null) {
+ om.setCertClient(certClient);
+ }
+ omList.add(om);
+
+ if (i <= numOfActiveOMs) {
+ om.start();
+ activeOMs.add(om);
+ LOG.info("Started OzoneManager RPC server at {}",
+ om.getOmRpcServerAddr());
+ } else {
+ inactiveOMs.add(om);
+ LOG.info("Intialized OzoneManager at {}. This OM is currently "
+ + "inactive (not running).", om.getOmRpcServerAddr());
+ }
+ }
+
+ // Set default OM address to point to the first OM. Clients would
+ // try connecting to this address by default
+ conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
+ NetUtils.getHostPortString(omList.get(0).getOmRpcServerAddr()));
+
+ break;
+ } catch (BindException e) {
+ for (OzoneManager om : omList) {
+ om.stop();
+ om.join();
+ LOG.info("Stopping OzoneManager server at {}",
+ om.getOmRpcServerAddr());
+ }
+ omList.clear();
+ ++retryCount;
+ LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
+ retryCount);
+ }
+ }
+ return omList;
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<StorageContainerManager> createSCMService()
+ throws IOException, AuthenticationException {
+ List<StorageContainerManager> scmList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 12000;
+
+ while (true) {
+ try {
+ basePort = 12000 + RANDOM.nextInt(1000) * 4;
+ initSCMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfSCMs; i++) {
+ // Set nodeId
+ String nodeId = SCM_NODE_ID_PREFIX + i;
+ String metaDirPath = path + "/" + nodeId;
+ OzoneConfiguration scmConfig = new OzoneConfiguration(conf);
+ scmConfig.set(OZONE_METADATA_DIRS, metaDirPath);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_NODE_ID_KEY, nodeId);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "");
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_KEY, "");
+ scmConfig.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+
+ // TODO: set SCM HA configs
+
+ configureSCM();
+ if (i == 1) {
+ StorageContainerManager.scmInit(scmConfig, clusterId);
+ } else {
+ StorageContainerManager.scmBootstrap(scmConfig);
+ }
+ StorageContainerManager scm = TestUtils.getScmSimple(scmConfig);
+ HealthyPipelineSafeModeRule rule =
+ scm.getScmSafeModeManager().getHealthyPipelineSafeModeRule();
+ if (rule != null) {
+ // Set threshold to wait for safe mode exit -
+ // this is needed since a pipeline is marked open only after
+ // leader election.
+ rule.setHealthyPipelineThresholdCount(numOfDatanodes / 3);
+ }
+ scmList.add(scm);
+
+ if (i <= numOfActiveSCMs) {
+ scm.start();
+ activeSCMs.add(scm);
+ LOG.info("Started SCM RPC server at {}",
+ scm.getClientProtocolServer());
+ } else {
+ inactiveSCMs.add(scm);
+ LOG.info("Intialized SCM at {}. This SCM is currently "
+ + "inactive (not running).", scm.getClientProtocolServer());
+ }
+ }
+
+
+ break;
+ } catch (BindException e) {
+ for (StorageContainerManager scm : scmList) {
+ scm.stop();
+ scm.join();
+ LOG.info("Stopping StorageContainerManager server at {}",
+ scm.getClientProtocolServer());
+ }
+ scmList.clear();
+ ++retryCount;
+ LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
+ retryCount);
+ }
+ }
+ return scmList;
+ }
+
+ protected void configureSCM() {
+// conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
+// conf.set(
+// ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
+// conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0");
+// conf.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "127.0.0.1:0");
+// conf.setInt(
+// ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY, numOfScmHandlers);
+// conf.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
+// "3s");
+// configureSCMheartbeat();
+ }
+
+ /**
+ * Initialize HA related configurations.
+ */
+ private void initSCMHAConfig(int basePort) throws IOException {
+ // Set configurations required for starting OM HA service, because that
+ // is the serviceID being passed to start Ozone HA cluster.
+ // Here setting internal service and OZONE_OM_SERVICE_IDS_KEY, in this
+ // way in OM start it uses internal service id to find it's service id.
+ conf.set(ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY, scmServiceId);
+ conf.set(ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID, scmServiceId);
+ String scmNodesKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_NODES_KEY, scmServiceId);
+ StringBuilder scmNodesKeyValue = new StringBuilder();
+ StringBuilder scmNames = new StringBuilder();
+
+ int port = basePort;
+
+ for (int i = 1; i <= numOfSCMs; i++, port+=10) {
+ String scmNodeId = SCM_NODE_ID_PREFIX + i;
+ scmNodesKeyValue.append(",").append(scmNodeId);
+ String scmAddrKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_ADDRESS_KEY, scmServiceId, scmNodeId);
+ String scmHttpAddrKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, scmServiceId, scmNodeId);
+ String scmHttpsAddrKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_KEY, scmServiceId, scmNodeId);
+ String scmRatisPortKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_RATIS_PORT_KEY, scmServiceId, scmNodeId);
+ String dnPortKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY,
+ scmServiceId, scmNodeId);
+ String blockClientKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+ scmServiceId, scmNodeId);
+ String ssClientKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY,
+ scmServiceId, scmNodeId);
+ String scmGrpcPortKey = ConfUtils.addKeySuffixes(
+ ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY, scmServiceId, scmNodeId);
+
+ conf.set(scmAddrKey, "127.0.0.1");
+ conf.set(scmHttpAddrKey, "127.0.0.1:" + (port + 2));
+ conf.set(scmHttpsAddrKey, "127.0.0.1:" + (port + 3));
+ conf.setInt(scmRatisPortKey, port + 4);
+ //conf.setInt("ozone.scm.ha.ratis.bind.port", port + 4);
+ conf.set(dnPortKey, "127.0.0.1:" + (port + 5));
+ conf.set(blockClientKey, "127.0.0.1:" + (port + 6));
+ conf.set(ssClientKey, "127.0.0.1:" + (port + 7));
+ //conf.setInt("ozone.scm.ha.ratis.grpc.bind.port", port + 8);
+ conf.setInt(scmGrpcPortKey, port + 8);
+ scmNames.append(",").append("localhost:" + (port + 5));
+ conf.set(ScmConfigKeys.
Review comment:
Minor: I think not required
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
##########
@@ -0,0 +1,738 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.safemode.HealthyPipelineSafeModeRule;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.recon.ReconServer;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.BindException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * MiniOzoneHAClusterImpl creates a complete in-process Ozone cluster
+ * with OM HA and SCM HA suitable for running tests.
+ * The cluster consists of a set of
+ * OzoneManagers, StorageContainerManagers and multiple DataNodes.
+ */
+public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MiniOzoneHAClusterImpl.class);
+
+ private final OMHAService omhaService;
+ private final SCMHAService scmhaService;
+ private static ObjectStore store = null;
+
+ private int waitForClusterToBeReadyTimeout = 120000; // 2 min
+
+ private static final Random RANDOM = new Random();
+ private static final int RATIS_RPC_TIMEOUT = 1000; // 1 second
+ public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
+
+ /**
+ * Creates a new MiniOzoneCluster.
+ *
+ * @throws IOException if there is an I/O error
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> activeOMList,
+ List<OzoneManager> inactiveOMList,
+ List<StorageContainerManager> activeSCMList,
+ List<StorageContainerManager> inactiveSCMList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId,
+ ReconServer reconServer) {
+ super(conf, hddsDatanodes, reconServer);
+ omhaService =
+ new OMHAService(activeOMList, inactiveOMList, omServiceId);
+ scmhaService =
+ new SCMHAService(activeSCMList, inactiveSCMList, scmServiceId);
+ }
+
+ /**
+ * Creates a new MiniOzoneCluster with all OMs active.
+ * This is used by MiniOzoneChaosCluster.
+ */
+ protected MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> omList,
+ List<StorageContainerManager> scmList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId) {
+ this(conf, omList, null, scmList, null, hddsDatanodes,
+ omServiceId, scmServiceId, null);
+ }
+
+ @Override
+ public String getOMServiceId() {
+ return omhaService.getServiceId();
+ }
+
+ @Override
+ public String getSCMServiceId() {
+ return scmhaService.getServiceId();
+ }
+
+ /**
+ * Returns the first OzoneManager from the list.
+ * @return
+ */
+ @Override
+ public OzoneManager getOzoneManager() {
+ return this.omhaService.getServices().get(0);
+ }
+
+ @Override
+ public OzoneClient getRpcClient() throws IOException {
+ String omServiceId = omhaService.getServiceId();
+ if (omServiceId == null) {
+ // Non-HA cluster.
+ return OzoneClientFactory.getRpcClient(getConf());
+ } else {
+ // HA cluster
+ return OzoneClientFactory.getRpcClient(omServiceId, getConf());
+ }
+ }
+
+ public boolean isOMActive(String omNodeId) {
+ return omhaService.isServiceActive(omNodeId);
+ }
+
+ public OzoneManager getOzoneManager(int index) {
+ return this.omhaService.getServiceByIndex(index);
+ }
+
+ public OzoneManager getOzoneManager(String omNodeId) {
+ return this.omhaService.getServiceById(omNodeId);
+ }
+
+ public List<OzoneManager> getOzoneManagersList() {
+ return omhaService.getServices();
+ }
+
+ /**
+ * Get OzoneManager leader object.
+ * @return OzoneManager object, null if there isn't one or more than one
+ */
+ public OzoneManager getOMLeader() {
+ OzoneManager res = null;
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager.isLeaderReady()) {
+ if (res != null) {
+ // Found more than one leader
+ // Return null, expect the caller to retry in a while
+ return null;
+ }
+ // Found a leader
+ res = ozoneManager;
+ }
+ }
+ return res;
+ }
+
+ /**
+ * Start a previously inactive OM.
+ */
+ public void startInactiveOM(String omNodeID) throws IOException {
+ omhaService.startInactiveService(omNodeID, OzoneManager::start);
+ }
+
+ @Override
+ public void restartOzoneManager() throws IOException {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ ozoneManager.stop();
+ ozoneManager.restart();
+ }
+ }
+
+ public void shutdownOzoneManager(OzoneManager ozoneManager) {
+ LOG.info("Shutting down OzoneManager " + ozoneManager.getOMNodeId());
+
+ ozoneManager.stop();
+ }
+
+ public void restartOzoneManager(OzoneManager ozoneManager, boolean waitForOM)
+ throws IOException, TimeoutException, InterruptedException {
+ LOG.info("Restarting OzoneManager " + ozoneManager.getOMNodeId());
+ ozoneManager.restart();
+
+ if (waitForOM) {
+ GenericTestUtils.waitFor(ozoneManager::isRunning,
+ 1000, waitForClusterToBeReadyTimeout);
+ }
+ }
+
+ public String getClusterId() throws IOException {
+ return scmhaService.getServices().get(0)
+ .getClientProtocolServer().getScmInfo().getClusterId();
+ }
+
+ public StorageContainerManager getActiveSCM() {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return scm;
+ }
+ }
+ return null;
+ }
+
+ public void waitForSCMToBeReady()
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(() -> {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return true;
+ }
+ }
+ return false;
+ }, 1000, waitForClusterToBeReadyTimeout);
+ }
+
+ @Override
+ public void stop() {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager != null) {
+ LOG.info("Stopping the OzoneManager {}", ozoneManager.getOMNodeId());
+ ozoneManager.stop();
+ ozoneManager.join();
+ }
+ }
+
+ for (StorageContainerManager scm : this.scmhaService.getServices()) {
+ if (scm != null) {
+ LOG.info("Stopping the StorageContainerManager {}", scm.getScmId());
+ scm.stop();
+ scm.join();
+ }
+ }
+ super.stop();
+ }
+
+ public void stopOzoneManager(int index) {
+ omhaService.getServices().get(index).stop();
+ omhaService.getServices().get(index).join();
+ }
+
+ public void stopOzoneManager(String omNodeId) {
+ omhaService.getServiceById(omNodeId).stop();
+ omhaService.getServiceById(omNodeId).join();
+ }
+
+ /**
+ * Builder for configuring the MiniOzoneCluster to run.
+ */
+ public static class Builder extends MiniOzoneClusterImpl.Builder {
+
+ private static final String OM_NODE_ID_PREFIX = "omNode-";
+ private List<OzoneManager> activeOMs = new ArrayList<>();
+ private List<OzoneManager> inactiveOMs = new ArrayList<>();
+
+ private static final String SCM_NODE_ID_PREFIX = "scmNode-";
+ private List<StorageContainerManager> activeSCMs = new ArrayList<>();
+ private List<StorageContainerManager> inactiveSCMs = new ArrayList<>();
+
+ /**
+ * Creates a new Builder.
+ *
+ * @param conf configuration
+ */
+ public Builder(OzoneConfiguration conf) {
+ super(conf);
+ }
+
+ public List<OzoneManager> getActiveOMs() {
+ return activeOMs;
+ }
+
+ public List<OzoneManager> getInactiveOMs() {
+ return inactiveOMs;
+ }
+
+ @Override
+ public MiniOzoneCluster build() throws IOException {
+ if (numOfActiveOMs > numOfOMs) {
+ throw new IllegalArgumentException("Number of active OMs cannot be " +
+ "more than the total number of OMs");
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveOMs == ACTIVE_OMS_NOT_SET) {
+ numOfActiveOMs = numOfOMs;
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveSCMs == ACTIVE_SCMS_NOT_SET) {
+ numOfActiveSCMs = numOfSCMs;
+ }
+
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ initializeConfiguration();
+ initOMRatisConf();
+ StorageContainerManager scm;
+ ReconServer reconServer = null;
+ try {
+ createSCMService();
+ createOMService();
+ if (includeRecon) {
+ configureRecon();
+ reconServer = new ReconServer();
+ reconServer.execute(new String[] {});
+ }
+ } catch (AuthenticationException ex) {
+ throw new IOException("Unable to build MiniOzoneCluster. ", ex);
+ }
+
+ final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(
+ activeSCMs, reconServer);
+
+ MiniOzoneHAClusterImpl cluster = new MiniOzoneHAClusterImpl(conf,
+ activeOMs, inactiveOMs, activeSCMs, inactiveSCMs,
+ hddsDatanodes, omServiceId, scmServiceId, reconServer);
+
+ if (startDataNodes) {
+ cluster.startHddsDatanodes();
+ }
+ return cluster;
+ }
+
+ protected void initOMRatisConf() {
+ conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
+ conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers);
+
+ // If test change the following config values we will respect,
+ // otherwise we will set lower timeout values.
+ long defaultDuration = OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT
+ .getDuration();
+ long curRatisRpcTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration == curRatisRpcTimeout ?
+ RATIS_RPC_TIMEOUT : curRatisRpcTimeout, TimeUnit.MILLISECONDS);
+
+ long defaultNodeFailureTimeout =
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getDuration();
+ long curNodeFailureTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ defaultNodeFailureTimeout,
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getUnit());
+ conf.setTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ curNodeFailureTimeout == defaultNodeFailureTimeout ?
+ NODE_FAILURE_TIMEOUT : curNodeFailureTimeout,
+ TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<OzoneManager> createOMService() throws IOException,
+ AuthenticationException {
+
+ List<OzoneManager> omList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 10000;
+
+ while (true) {
+ try {
+ basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ initOMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfOMs; i++) {
+ // Set nodeId
+ String nodeId = OM_NODE_ID_PREFIX + i;
+ OzoneConfiguration config = new OzoneConfiguration(conf);
+ config.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeId);
+ // Set the OM http(s) address to null so that the cluster picks
+ // up the address set with service ID and node ID in initHAConfig
+ config.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "");
+ config.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, "");
+
+ // Set metadata/DB dir base path
+ String metaDirPath = path + "/" + nodeId;
+ config.set(OZONE_METADATA_DIRS, metaDirPath);
+ // OMStorage omStore = new OMStorage(config);
+ // initializeOmStorage(omStore);
+ OzoneManager.omInit(config);
+ OzoneManager om = OzoneManager.createOm(config);
+ if (certClient != null) {
+ om.setCertClient(certClient);
+ }
+ omList.add(om);
+
+ if (i <= numOfActiveOMs) {
+ om.start();
+ activeOMs.add(om);
+ LOG.info("Started OzoneManager RPC server at {}",
+ om.getOmRpcServerAddr());
+ } else {
+ inactiveOMs.add(om);
+ LOG.info("Intialized OzoneManager at {}. This OM is currently "
+ + "inactive (not running).", om.getOmRpcServerAddr());
+ }
+ }
+
+ // Set default OM address to point to the first OM. Clients would
+ // try connecting to this address by default
+ conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
+ NetUtils.getHostPortString(omList.get(0).getOmRpcServerAddr()));
+
+ break;
+ } catch (BindException e) {
+ for (OzoneManager om : omList) {
+ om.stop();
+ om.join();
+ LOG.info("Stopping OzoneManager server at {}",
+ om.getOmRpcServerAddr());
+ }
+ omList.clear();
+ ++retryCount;
+ LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
+ retryCount);
+ }
+ }
+ return omList;
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<StorageContainerManager> createSCMService()
+ throws IOException, AuthenticationException {
+ List<StorageContainerManager> scmList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 12000;
+
+ while (true) {
+ try {
+ basePort = 12000 + RANDOM.nextInt(1000) * 4;
+ initSCMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfSCMs; i++) {
+ // Set nodeId
+ String nodeId = SCM_NODE_ID_PREFIX + i;
+ String metaDirPath = path + "/" + nodeId;
+ OzoneConfiguration scmConfig = new OzoneConfiguration(conf);
+ scmConfig.set(OZONE_METADATA_DIRS, metaDirPath);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_NODE_ID_KEY, nodeId);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "");
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_KEY, "");
+ scmConfig.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+
+ // TODO: set SCM HA configs
+
+ configureSCM();
+ if (i == 1) {
+ StorageContainerManager.scmInit(scmConfig, clusterId);
+ } else {
+ StorageContainerManager.scmBootstrap(scmConfig);
+ }
+ StorageContainerManager scm = TestUtils.getScmSimple(scmConfig);
+ HealthyPipelineSafeModeRule rule =
+ scm.getScmSafeModeManager().getHealthyPipelineSafeModeRule();
+ if (rule != null) {
+ // Set threshold to wait for safe mode exit -
+ // this is needed since a pipeline is marked open only after
+ // leader election.
+ rule.setHealthyPipelineThresholdCount(numOfDatanodes / 3);
+ }
+ scmList.add(scm);
+
+ if (i <= numOfActiveSCMs) {
+ scm.start();
+ activeSCMs.add(scm);
+ LOG.info("Started SCM RPC server at {}",
+ scm.getClientProtocolServer());
+ } else {
+ inactiveSCMs.add(scm);
+ LOG.info("Intialized SCM at {}. This SCM is currently "
+ + "inactive (not running).", scm.getClientProtocolServer());
+ }
+ }
+
+
+ break;
+ } catch (BindException e) {
+ for (StorageContainerManager scm : scmList) {
+ scm.stop();
+ scm.join();
+ LOG.info("Stopping StorageContainerManager server at {}",
+ scm.getClientProtocolServer());
+ }
+ scmList.clear();
+ ++retryCount;
+ LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
+ retryCount);
+ }
+ }
+ return scmList;
+ }
+
+ protected void configureSCM() {
Review comment:
We can remove the commented code.
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
##########
@@ -0,0 +1,738 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.safemode.HealthyPipelineSafeModeRule;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.recon.ReconServer;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.BindException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * MiniOzoneHAClusterImpl creates a complete in-process Ozone cluster
+ * with OM HA and SCM HA suitable for running tests.
+ * The cluster consists of a set of
+ * OzoneManagers, StorageContainerManagers and multiple DataNodes.
+ */
+public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MiniOzoneHAClusterImpl.class);
+
+ private final OMHAService omhaService;
+ private final SCMHAService scmhaService;
+ private static ObjectStore store = null;
+
+ private int waitForClusterToBeReadyTimeout = 120000; // 2 min
+
+ private static final Random RANDOM = new Random();
+ private static final int RATIS_RPC_TIMEOUT = 1000; // 1 second
+ public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
+
+ /**
+ * Creates a new MiniOzoneCluster.
+ *
+ * @throws IOException if there is an I/O error
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> activeOMList,
+ List<OzoneManager> inactiveOMList,
+ List<StorageContainerManager> activeSCMList,
+ List<StorageContainerManager> inactiveSCMList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId,
+ ReconServer reconServer) {
+ super(conf, hddsDatanodes, reconServer);
+ omhaService =
+ new OMHAService(activeOMList, inactiveOMList, omServiceId);
+ scmhaService =
+ new SCMHAService(activeSCMList, inactiveSCMList, scmServiceId);
+ }
+
+ /**
+ * Creates a new MiniOzoneCluster with all OMs active.
+ * This is used by MiniOzoneChaosCluster.
+ */
+ protected MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> omList,
+ List<StorageContainerManager> scmList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId) {
+ this(conf, omList, null, scmList, null, hddsDatanodes,
+ omServiceId, scmServiceId, null);
+ }
+
+ @Override
+ public String getOMServiceId() {
+ return omhaService.getServiceId();
+ }
+
+ @Override
+ public String getSCMServiceId() {
+ return scmhaService.getServiceId();
+ }
+
+ /**
+ * Returns the first OzoneManager from the list.
+ * @return
+ */
+ @Override
+ public OzoneManager getOzoneManager() {
+ return this.omhaService.getServices().get(0);
+ }
+
+ @Override
+ public OzoneClient getRpcClient() throws IOException {
+ String omServiceId = omhaService.getServiceId();
+ if (omServiceId == null) {
+ // Non-HA cluster.
+ return OzoneClientFactory.getRpcClient(getConf());
+ } else {
+ // HA cluster
+ return OzoneClientFactory.getRpcClient(omServiceId, getConf());
+ }
+ }
+
+ public boolean isOMActive(String omNodeId) {
+ return omhaService.isServiceActive(omNodeId);
+ }
+
+ public OzoneManager getOzoneManager(int index) {
+ return this.omhaService.getServiceByIndex(index);
+ }
+
+ public OzoneManager getOzoneManager(String omNodeId) {
+ return this.omhaService.getServiceById(omNodeId);
+ }
+
+ public List<OzoneManager> getOzoneManagersList() {
+ return omhaService.getServices();
+ }
+
+ /**
+ * Get OzoneManager leader object.
+ * @return OzoneManager object, null if there isn't one or more than one
+ */
+ public OzoneManager getOMLeader() {
+ OzoneManager res = null;
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager.isLeaderReady()) {
+ if (res != null) {
+ // Found more than one leader
+ // Return null, expect the caller to retry in a while
+ return null;
+ }
+ // Found a leader
+ res = ozoneManager;
+ }
+ }
+ return res;
+ }
+
+ /**
+ * Start a previously inactive OM.
+ */
+ public void startInactiveOM(String omNodeID) throws IOException {
+ omhaService.startInactiveService(omNodeID, OzoneManager::start);
+ }
+
+ @Override
+ public void restartOzoneManager() throws IOException {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ ozoneManager.stop();
+ ozoneManager.restart();
+ }
+ }
+
+ public void shutdownOzoneManager(OzoneManager ozoneManager) {
+ LOG.info("Shutting down OzoneManager " + ozoneManager.getOMNodeId());
+
+ ozoneManager.stop();
+ }
+
+ public void restartOzoneManager(OzoneManager ozoneManager, boolean waitForOM)
+ throws IOException, TimeoutException, InterruptedException {
+ LOG.info("Restarting OzoneManager " + ozoneManager.getOMNodeId());
+ ozoneManager.restart();
+
+ if (waitForOM) {
+ GenericTestUtils.waitFor(ozoneManager::isRunning,
+ 1000, waitForClusterToBeReadyTimeout);
+ }
+ }
+
+ public String getClusterId() throws IOException {
+ return scmhaService.getServices().get(0)
+ .getClientProtocolServer().getScmInfo().getClusterId();
+ }
+
+ public StorageContainerManager getActiveSCM() {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return scm;
+ }
+ }
+ return null;
+ }
+
+ public void waitForSCMToBeReady()
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(() -> {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return true;
+ }
+ }
+ return false;
+ }, 1000, waitForClusterToBeReadyTimeout);
+ }
+
+ @Override
+ public void stop() {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager != null) {
+ LOG.info("Stopping the OzoneManager {}", ozoneManager.getOMNodeId());
+ ozoneManager.stop();
+ ozoneManager.join();
+ }
+ }
+
+ for (StorageContainerManager scm : this.scmhaService.getServices()) {
+ if (scm != null) {
+ LOG.info("Stopping the StorageContainerManager {}", scm.getScmId());
+ scm.stop();
+ scm.join();
+ }
+ }
+ super.stop();
+ }
+
+ public void stopOzoneManager(int index) {
+ omhaService.getServices().get(index).stop();
+ omhaService.getServices().get(index).join();
+ }
+
+ public void stopOzoneManager(String omNodeId) {
+ omhaService.getServiceById(omNodeId).stop();
+ omhaService.getServiceById(omNodeId).join();
+ }
+
+ /**
+ * Builder for configuring the MiniOzoneCluster to run.
+ */
+ public static class Builder extends MiniOzoneClusterImpl.Builder {
+
+ private static final String OM_NODE_ID_PREFIX = "omNode-";
+ private List<OzoneManager> activeOMs = new ArrayList<>();
+ private List<OzoneManager> inactiveOMs = new ArrayList<>();
+
+ private static final String SCM_NODE_ID_PREFIX = "scmNode-";
+ private List<StorageContainerManager> activeSCMs = new ArrayList<>();
+ private List<StorageContainerManager> inactiveSCMs = new ArrayList<>();
+
+ /**
+ * Creates a new Builder.
+ *
+ * @param conf configuration
+ */
+ public Builder(OzoneConfiguration conf) {
+ super(conf);
+ }
+
+ public List<OzoneManager> getActiveOMs() {
+ return activeOMs;
+ }
+
+ public List<OzoneManager> getInactiveOMs() {
+ return inactiveOMs;
+ }
+
+ @Override
+ public MiniOzoneCluster build() throws IOException {
+ if (numOfActiveOMs > numOfOMs) {
+ throw new IllegalArgumentException("Number of active OMs cannot be " +
+ "more than the total number of OMs");
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveOMs == ACTIVE_OMS_NOT_SET) {
+ numOfActiveOMs = numOfOMs;
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveSCMs == ACTIVE_SCMS_NOT_SET) {
+ numOfActiveSCMs = numOfSCMs;
+ }
+
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ initializeConfiguration();
+ initOMRatisConf();
+ StorageContainerManager scm;
+ ReconServer reconServer = null;
+ try {
+ createSCMService();
+ createOMService();
+ if (includeRecon) {
+ configureRecon();
+ reconServer = new ReconServer();
+ reconServer.execute(new String[] {});
+ }
+ } catch (AuthenticationException ex) {
+ throw new IOException("Unable to build MiniOzoneCluster. ", ex);
+ }
+
+ final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(
+ activeSCMs, reconServer);
+
+ MiniOzoneHAClusterImpl cluster = new MiniOzoneHAClusterImpl(conf,
+ activeOMs, inactiveOMs, activeSCMs, inactiveSCMs,
+ hddsDatanodes, omServiceId, scmServiceId, reconServer);
+
+ if (startDataNodes) {
+ cluster.startHddsDatanodes();
+ }
+ return cluster;
+ }
+
+ protected void initOMRatisConf() {
+ conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
+ conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers);
+
+ // If test change the following config values we will respect,
+ // otherwise we will set lower timeout values.
+ long defaultDuration = OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT
+ .getDuration();
+ long curRatisRpcTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration == curRatisRpcTimeout ?
+ RATIS_RPC_TIMEOUT : curRatisRpcTimeout, TimeUnit.MILLISECONDS);
+
+ long defaultNodeFailureTimeout =
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getDuration();
+ long curNodeFailureTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ defaultNodeFailureTimeout,
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getUnit());
+ conf.setTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ curNodeFailureTimeout == defaultNodeFailureTimeout ?
+ NODE_FAILURE_TIMEOUT : curNodeFailureTimeout,
+ TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<OzoneManager> createOMService() throws IOException,
+ AuthenticationException {
+
+ List<OzoneManager> omList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 10000;
+
+ while (true) {
+ try {
+ basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ initOMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfOMs; i++) {
+ // Set nodeId
+ String nodeId = OM_NODE_ID_PREFIX + i;
+ OzoneConfiguration config = new OzoneConfiguration(conf);
+ config.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeId);
+ // Set the OM http(s) address to null so that the cluster picks
+ // up the address set with service ID and node ID in initHAConfig
+ config.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "");
+ config.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, "");
+
+ // Set metadata/DB dir base path
+ String metaDirPath = path + "/" + nodeId;
+ config.set(OZONE_METADATA_DIRS, metaDirPath);
+ // OMStorage omStore = new OMStorage(config);
+ // initializeOmStorage(omStore);
+ OzoneManager.omInit(config);
+ OzoneManager om = OzoneManager.createOm(config);
+ if (certClient != null) {
+ om.setCertClient(certClient);
+ }
+ omList.add(om);
+
+ if (i <= numOfActiveOMs) {
+ om.start();
+ activeOMs.add(om);
+ LOG.info("Started OzoneManager RPC server at {}",
+ om.getOmRpcServerAddr());
+ } else {
+ inactiveOMs.add(om);
+ LOG.info("Intialized OzoneManager at {}. This OM is currently "
+ + "inactive (not running).", om.getOmRpcServerAddr());
+ }
+ }
+
+ // Set default OM address to point to the first OM. Clients would
+ // try connecting to this address by default
+ conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
+ NetUtils.getHostPortString(omList.get(0).getOmRpcServerAddr()));
+
+ break;
+ } catch (BindException e) {
+ for (OzoneManager om : omList) {
+ om.stop();
+ om.join();
+ LOG.info("Stopping OzoneManager server at {}",
+ om.getOmRpcServerAddr());
+ }
+ omList.clear();
+ ++retryCount;
+ LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
+ retryCount);
+ }
+ }
+ return omList;
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<StorageContainerManager> createSCMService()
+ throws IOException, AuthenticationException {
+ List<StorageContainerManager> scmList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 12000;
+
+ while (true) {
+ try {
+ basePort = 12000 + RANDOM.nextInt(1000) * 4;
+ initSCMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfSCMs; i++) {
+ // Set nodeId
+ String nodeId = SCM_NODE_ID_PREFIX + i;
+ String metaDirPath = path + "/" + nodeId;
+ OzoneConfiguration scmConfig = new OzoneConfiguration(conf);
+ scmConfig.set(OZONE_METADATA_DIRS, metaDirPath);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_NODE_ID_KEY, nodeId);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "");
Review comment:
Minor: We don't need lines 470,471
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
##########
@@ -0,0 +1,738 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.safemode.HealthyPipelineSafeModeRule;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.recon.ReconServer;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.BindException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * MiniOzoneHAClusterImpl creates a complete in-process Ozone cluster
+ * with OM HA and SCM HA suitable for running tests.
+ * The cluster consists of a set of
+ * OzoneManagers, StorageContainerManagers and multiple DataNodes.
+ */
+public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MiniOzoneHAClusterImpl.class);
+
+ private final OMHAService omhaService;
+ private final SCMHAService scmhaService;
+ private static ObjectStore store = null;
+
+ private int waitForClusterToBeReadyTimeout = 120000; // 2 min
+
+ private static final Random RANDOM = new Random();
+ private static final int RATIS_RPC_TIMEOUT = 1000; // 1 second
+ public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
+
+ /**
+ * Creates a new MiniOzoneCluster.
+ *
+ * @throws IOException if there is an I/O error
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> activeOMList,
+ List<OzoneManager> inactiveOMList,
+ List<StorageContainerManager> activeSCMList,
+ List<StorageContainerManager> inactiveSCMList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId,
+ ReconServer reconServer) {
+ super(conf, hddsDatanodes, reconServer);
+ omhaService =
+ new OMHAService(activeOMList, inactiveOMList, omServiceId);
+ scmhaService =
+ new SCMHAService(activeSCMList, inactiveSCMList, scmServiceId);
+ }
+
+ /**
+ * Creates a new MiniOzoneCluster with all OMs active.
+ * This is used by MiniOzoneChaosCluster.
+ */
+ protected MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> omList,
+ List<StorageContainerManager> scmList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId) {
+ this(conf, omList, null, scmList, null, hddsDatanodes,
+ omServiceId, scmServiceId, null);
+ }
+
+ @Override
+ public String getOMServiceId() {
+ return omhaService.getServiceId();
+ }
+
+ @Override
+ public String getSCMServiceId() {
+ return scmhaService.getServiceId();
+ }
+
+ /**
+ * Returns the first OzoneManager from the list.
+ * @return
+ */
+ @Override
+ public OzoneManager getOzoneManager() {
+ return this.omhaService.getServices().get(0);
+ }
+
+ @Override
+ public OzoneClient getRpcClient() throws IOException {
+ String omServiceId = omhaService.getServiceId();
+ if (omServiceId == null) {
+ // Non-HA cluster.
+ return OzoneClientFactory.getRpcClient(getConf());
+ } else {
+ // HA cluster
+ return OzoneClientFactory.getRpcClient(omServiceId, getConf());
+ }
+ }
+
+ public boolean isOMActive(String omNodeId) {
+ return omhaService.isServiceActive(omNodeId);
+ }
+
+ public OzoneManager getOzoneManager(int index) {
+ return this.omhaService.getServiceByIndex(index);
+ }
+
+ public OzoneManager getOzoneManager(String omNodeId) {
+ return this.omhaService.getServiceById(omNodeId);
+ }
+
+ public List<OzoneManager> getOzoneManagersList() {
+ return omhaService.getServices();
+ }
+
+ /**
+ * Get OzoneManager leader object.
+ * @return OzoneManager object, null if there isn't one or more than one
+ */
+ public OzoneManager getOMLeader() {
+ OzoneManager res = null;
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager.isLeaderReady()) {
+ if (res != null) {
+ // Found more than one leader
+ // Return null, expect the caller to retry in a while
+ return null;
+ }
+ // Found a leader
+ res = ozoneManager;
+ }
+ }
+ return res;
+ }
+
+ /**
+ * Start a previously inactive OM.
+ */
+ public void startInactiveOM(String omNodeID) throws IOException {
+ omhaService.startInactiveService(omNodeID, OzoneManager::start);
+ }
+
+ @Override
+ public void restartOzoneManager() throws IOException {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ ozoneManager.stop();
+ ozoneManager.restart();
+ }
+ }
+
+ public void shutdownOzoneManager(OzoneManager ozoneManager) {
+ LOG.info("Shutting down OzoneManager " + ozoneManager.getOMNodeId());
+
+ ozoneManager.stop();
+ }
+
+ public void restartOzoneManager(OzoneManager ozoneManager, boolean waitForOM)
+ throws IOException, TimeoutException, InterruptedException {
+ LOG.info("Restarting OzoneManager " + ozoneManager.getOMNodeId());
+ ozoneManager.restart();
+
+ if (waitForOM) {
+ GenericTestUtils.waitFor(ozoneManager::isRunning,
+ 1000, waitForClusterToBeReadyTimeout);
+ }
+ }
+
+ public String getClusterId() throws IOException {
+ return scmhaService.getServices().get(0)
+ .getClientProtocolServer().getScmInfo().getClusterId();
+ }
+
+ public StorageContainerManager getActiveSCM() {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return scm;
+ }
+ }
+ return null;
+ }
+
+ public void waitForSCMToBeReady()
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(() -> {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return true;
+ }
+ }
+ return false;
+ }, 1000, waitForClusterToBeReadyTimeout);
+ }
+
+ @Override
+ public void stop() {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager != null) {
+ LOG.info("Stopping the OzoneManager {}", ozoneManager.getOMNodeId());
+ ozoneManager.stop();
+ ozoneManager.join();
+ }
+ }
+
+ for (StorageContainerManager scm : this.scmhaService.getServices()) {
+ if (scm != null) {
+ LOG.info("Stopping the StorageContainerManager {}", scm.getScmId());
+ scm.stop();
+ scm.join();
+ }
+ }
+ super.stop();
+ }
+
+ public void stopOzoneManager(int index) {
+ omhaService.getServices().get(index).stop();
+ omhaService.getServices().get(index).join();
+ }
+
+ public void stopOzoneManager(String omNodeId) {
+ omhaService.getServiceById(omNodeId).stop();
+ omhaService.getServiceById(omNodeId).join();
+ }
+
+ /**
+ * Builder for configuring the MiniOzoneCluster to run.
+ */
+ public static class Builder extends MiniOzoneClusterImpl.Builder {
+
+ private static final String OM_NODE_ID_PREFIX = "omNode-";
+ private List<OzoneManager> activeOMs = new ArrayList<>();
+ private List<OzoneManager> inactiveOMs = new ArrayList<>();
+
+ private static final String SCM_NODE_ID_PREFIX = "scmNode-";
+ private List<StorageContainerManager> activeSCMs = new ArrayList<>();
+ private List<StorageContainerManager> inactiveSCMs = new ArrayList<>();
+
+ /**
+ * Creates a new Builder.
+ *
+ * @param conf configuration
+ */
+ public Builder(OzoneConfiguration conf) {
+ super(conf);
+ }
+
+ public List<OzoneManager> getActiveOMs() {
+ return activeOMs;
+ }
+
+ public List<OzoneManager> getInactiveOMs() {
+ return inactiveOMs;
+ }
+
+ @Override
+ public MiniOzoneCluster build() throws IOException {
+ if (numOfActiveOMs > numOfOMs) {
+ throw new IllegalArgumentException("Number of active OMs cannot be " +
+ "more than the total number of OMs");
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveOMs == ACTIVE_OMS_NOT_SET) {
+ numOfActiveOMs = numOfOMs;
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveSCMs == ACTIVE_SCMS_NOT_SET) {
+ numOfActiveSCMs = numOfSCMs;
+ }
+
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ initializeConfiguration();
+ initOMRatisConf();
+ StorageContainerManager scm;
+ ReconServer reconServer = null;
+ try {
+ createSCMService();
+ createOMService();
+ if (includeRecon) {
+ configureRecon();
+ reconServer = new ReconServer();
+ reconServer.execute(new String[] {});
+ }
+ } catch (AuthenticationException ex) {
+ throw new IOException("Unable to build MiniOzoneCluster. ", ex);
+ }
+
+ final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(
+ activeSCMs, reconServer);
+
+ MiniOzoneHAClusterImpl cluster = new MiniOzoneHAClusterImpl(conf,
+ activeOMs, inactiveOMs, activeSCMs, inactiveSCMs,
+ hddsDatanodes, omServiceId, scmServiceId, reconServer);
+
+ if (startDataNodes) {
+ cluster.startHddsDatanodes();
+ }
+ return cluster;
+ }
+
+ protected void initOMRatisConf() {
+ conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
+ conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers);
+
+ // If test change the following config values we will respect,
+ // otherwise we will set lower timeout values.
+ long defaultDuration = OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT
+ .getDuration();
+ long curRatisRpcTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration == curRatisRpcTimeout ?
+ RATIS_RPC_TIMEOUT : curRatisRpcTimeout, TimeUnit.MILLISECONDS);
+
+ long defaultNodeFailureTimeout =
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getDuration();
+ long curNodeFailureTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ defaultNodeFailureTimeout,
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getUnit());
+ conf.setTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ curNodeFailureTimeout == defaultNodeFailureTimeout ?
+ NODE_FAILURE_TIMEOUT : curNodeFailureTimeout,
+ TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<OzoneManager> createOMService() throws IOException,
+ AuthenticationException {
+
+ List<OzoneManager> omList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 10000;
+
+ while (true) {
+ try {
+ basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ initOMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfOMs; i++) {
+ // Set nodeId
+ String nodeId = OM_NODE_ID_PREFIX + i;
+ OzoneConfiguration config = new OzoneConfiguration(conf);
+ config.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeId);
+ // Set the OM http(s) address to null so that the cluster picks
+ // up the address set with service ID and node ID in initHAConfig
+ config.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "");
+ config.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, "");
+
+ // Set metadata/DB dir base path
+ String metaDirPath = path + "/" + nodeId;
+ config.set(OZONE_METADATA_DIRS, metaDirPath);
+ // OMStorage omStore = new OMStorage(config);
+ // initializeOmStorage(omStore);
+ OzoneManager.omInit(config);
+ OzoneManager om = OzoneManager.createOm(config);
+ if (certClient != null) {
+ om.setCertClient(certClient);
+ }
+ omList.add(om);
+
+ if (i <= numOfActiveOMs) {
+ om.start();
+ activeOMs.add(om);
+ LOG.info("Started OzoneManager RPC server at {}",
+ om.getOmRpcServerAddr());
+ } else {
+ inactiveOMs.add(om);
+ LOG.info("Intialized OzoneManager at {}. This OM is currently "
+ + "inactive (not running).", om.getOmRpcServerAddr());
+ }
+ }
+
+ // Set default OM address to point to the first OM. Clients would
+ // try connecting to this address by default
+ conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
+ NetUtils.getHostPortString(omList.get(0).getOmRpcServerAddr()));
+
+ break;
+ } catch (BindException e) {
+ for (OzoneManager om : omList) {
+ om.stop();
+ om.join();
+ LOG.info("Stopping OzoneManager server at {}",
+ om.getOmRpcServerAddr());
+ }
+ omList.clear();
+ ++retryCount;
+ LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
+ retryCount);
+ }
+ }
+ return omList;
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<StorageContainerManager> createSCMService()
+ throws IOException, AuthenticationException {
+ List<StorageContainerManager> scmList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 12000;
+
+ while (true) {
+ try {
+ basePort = 12000 + RANDOM.nextInt(1000) * 4;
+ initSCMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfSCMs; i++) {
+ // Set nodeId
+ String nodeId = SCM_NODE_ID_PREFIX + i;
+ String metaDirPath = path + "/" + nodeId;
+ OzoneConfiguration scmConfig = new OzoneConfiguration(conf);
+ scmConfig.set(OZONE_METADATA_DIRS, metaDirPath);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_NODE_ID_KEY, nodeId);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "");
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_KEY, "");
+ scmConfig.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+
+ // TODO: set SCM HA configs
+
+ configureSCM();
+ if (i == 1) {
+ StorageContainerManager.scmInit(scmConfig, clusterId);
+ } else {
+ StorageContainerManager.scmBootstrap(scmConfig);
+ }
+ StorageContainerManager scm = TestUtils.getScmSimple(scmConfig);
+ HealthyPipelineSafeModeRule rule =
+ scm.getScmSafeModeManager().getHealthyPipelineSafeModeRule();
+ if (rule != null) {
+ // Set threshold to wait for safe mode exit -
+ // this is needed since a pipeline is marked open only after
+ // leader election.
+ rule.setHealthyPipelineThresholdCount(numOfDatanodes / 3);
+ }
+ scmList.add(scm);
+
+ if (i <= numOfActiveSCMs) {
+ scm.start();
+ activeSCMs.add(scm);
+ LOG.info("Started SCM RPC server at {}",
+ scm.getClientProtocolServer());
+ } else {
+ inactiveSCMs.add(scm);
+ LOG.info("Intialized SCM at {}. This SCM is currently "
+ + "inactive (not running).", scm.getClientProtocolServer());
+ }
+ }
+
+
+ break;
+ } catch (BindException e) {
+ for (StorageContainerManager scm : scmList) {
+ scm.stop();
+ scm.join();
+ LOG.info("Stopping StorageContainerManager server at {}",
+ scm.getClientProtocolServer());
+ }
+ scmList.clear();
+ ++retryCount;
+ LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
+ retryCount);
+ }
+ }
+ return scmList;
+ }
+
+ protected void configureSCM() {
Review comment:
Don't we need to call configureSCMheartbeat which sets to 1sec?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r581928988
##########
File path: hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
##########
@@ -77,6 +78,11 @@
*/
ScmInfo getScmInfo() throws IOException;
+ /**
+ * Request to add SCM instance to HA group.
+ */
+ boolean addSCM(AddSCMRequest request) throws IOException;
Review comment:
i was thinking about doing this.. But seems, like OM also uses the block client protocol to connect to SCM. That's why reused it. We can discuss over this as we move along and conclude.
There has been discussions to merge the block protocol with container protocol as well.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583450436
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.Rule;
+
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
+
+/**
+ * Base class for Ozone Manager HA tests.
+ */
+public class TestStorageContainerManagerHA {
+
+ private MiniOzoneHAClusterImpl cluster = null;
+ private OzoneConfiguration conf;
+ private String clusterId;
+ private String scmId;
+ private String omServiceId;
+ private static int numOfOMs = 3;
+ private String scmServiceId;
+ // TODO: 3 servers will not work until SCM HA configs are considered in
+ // SCM HA Ratis server init
+ private static int numOfSCMs = 3;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omServiceId = "om-service-test1";
+ scmServiceId = "scm-service-test1";
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setScmId(scmId)
+ .setOMServiceId(omServiceId)
+ .setSCMServiceId(scmServiceId)
+ .setNumOfStorageContainerManagers(numOfSCMs)
+ .setNumOfOzoneManagers(numOfOMs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAllSCMAreRunning() throws Exception {
+ int count = 0;
+ List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
+ Assert.assertEquals(numOfSCMs, scms.size());
+ int peerSize = cluster.getStorageContainerManager().getScmHAManager()
+ .getRatisServer().getDivision().getGroup().getPeers().size();
+ for (StorageContainerManager scm : scms) {
+ if (scm.checkLeader()) {
+ count++;
+ }
+ Assert.assertTrue(peerSize == numOfSCMs);
+ }
+ Assert.assertEquals(1, count);
+ count = 0;
+ List<OzoneManager> oms = cluster.getOzoneManagersList();
+ Assert.assertEquals(numOfOMs, oms.size());
+ for (OzoneManager om : oms) {
+ if (om.isLeaderReady()) {
+ count++;
+ }
+ }
+ Assert.assertEquals(1, count);
+ testPutKey();
+ }
+
+ public void testPutKey() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ Instant testStartTime = Instant.now();
+ ObjectStore store =
+ OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length, STAND_ALONE, ONE,
+ new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
+ Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ is.close();
+ final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .setRefreshPipeline(true).build();
+ final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ final List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ final long containerID = keyLocationInfos.get(0).getContainerID();
+ for (int k = 0; k < numOfSCMs; k++) {
+ StorageContainerManager scm =
+ cluster.getStorageContainerManagers().get(k);
+ // flush to DB on each SCM
Review comment:
That's why explicit take snapshot call is introduced so that, it first waits for all the appended log entries to complete apply .
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] GlenGeng commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
GlenGeng commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r582600604
##########
File path: hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto
##########
@@ -54,6 +55,7 @@ message SCMBlockLocationRequest {
optional DeleteScmKeyBlocksRequestProto deleteScmKeyBlocksRequest = 12;
optional hadoop.hdds.GetScmInfoRequestProto getScmInfoRequest = 13;
optional SortDatanodesRequestProto sortDatanodesRequest = 14;
+ optional hadoop.hdds.AddScmRequestProto addScmRequestProto = 15;
Review comment:
indent
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -156,16 +200,52 @@ public void stop() throws IOException {
@Override
public NotLeaderException triggerNotLeaderException() {
return new NotLeaderException(
- division.getMemberId(), null, division.getGroup().getPeers());
+ getDivision().getMemberId(), null, getDivision().getGroup().getPeers());
+ }
+
+ @Override
+ public boolean addSCM(AddSCMRequest request) throws IOException {
+ List<RaftPeer> newRaftPeerList =
+ new ArrayList<>(getDivision().getGroup().getPeers());
+ // add the SCM node to be added to the raft peer list
+
+ RaftPeer raftPeer = RaftPeer.newBuilder().setId(request.getScmId())
+ .setAddress(request.getRatisAddr()).build();
+ newRaftPeerList.add(raftPeer);
+
+ LOG.debug("{}: Submitting SetConfiguration request to Ratis server with" +
+ " new SCM peers list: {}", scm.getScmId(),
+ newRaftPeerList);
+
+ RaftServer.Division division = getDivision();
+ SetConfigurationRequest configRequest =
+ new SetConfigurationRequest(clientId, division.getPeer().getId(),
+ division.getGroup().getGroupId(), nextCallId(), newRaftPeerList);
+
+ try {
+ RaftClientReply raftClientReply =
+ division.getRaftServer().setConfiguration(configRequest);
+ if (raftClientReply.isSuccess()) {
+ LOG.info("Successfully added new SCM: {}.", request.getScmId());
+ } else {
+ LOG.error("Failed to add new SCM: {}. Ratis reply: {}" +
+ request.getScmId(), raftClientReply);
Review comment:
Need throw exception to trigger failover proxy to try next SCM.
fpp consider a response with a false status to be a successful response.
##########
File path: hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
##########
@@ -131,4 +134,24 @@ public static String getScmServiceId(ConfigurationSource conf) {
}
return localScmServiceId;
}
+
+ public static OzoneConfiguration removeSelfId(
+ OzoneConfiguration configuration) {
+ final OzoneConfiguration conf = new OzoneConfiguration(configuration);
+ String scmNodes = conf.get(ConfUtils
+ .addKeySuffixes(ScmConfigKeys.OZONE_SCM_NODES_KEY,
+ getScmServiceId(conf)));
+ if (scmNodes != null) {
+ final String selfId = conf.get(ScmConfigKeys.OZONE_SCM_NODE_ID_KEY);
+ String[] parts = scmNodes.split(",");
+ List<String> partList = new ArrayList<>();
Review comment:
partList -> partsLeft
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java
##########
@@ -55,7 +56,8 @@
public InterSCMGrpcClient(final String host, final ConfigurationSource conf) {
Preconditions.checkNotNull(conf);
- int port = conf.getObject(SCMHAConfiguration.class).getGrpcBindPort();
Review comment:
will you remove
```
@Config(key = "grpc.bind.port",
type = ConfigType.INT,
defaultValue = "9899",
tags = {OZONE, SCM, HA, RATIS},
description = "Port used by SCM for Grpc Server."
)
private int grpcBindPort = 9866;
```
from SCMHAConfiguration ?
##########
File path: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
##########
@@ -765,12 +765,18 @@ public void addGroup(HddsProtos.PipelineID pipelineId,
clientId, server.getId(), nextCallId(), group);
RaftClientReply reply;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received create pipeline request for pipeline {}", pipelineID);
Review comment:
`create pipeline request` seem not accurate
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] vivekratnavel commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
vivekratnavel commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r582213851
##########
File path: hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
##########
@@ -311,8 +311,8 @@ public static InetSocketAddress getReconAddresses(
public static InetSocketAddress getSingleSCMAddress(
ConfigurationSource conf) {
Collection<InetSocketAddress> singleton = getSCMAddresses(conf);
- Preconditions.checkArgument(singleton.size() == 1,
- MULTIPLE_SCM_NOT_YET_SUPPORTED);
+ // Preconditions.checkArgument(singleton.size() == 1,
Review comment:
Can we remove this?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#issuecomment-787538429
Thanks @bshashikant , @vivekratnavel . @GlenGeng for the reviews. I have committed this.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583571142
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
+ server.close();
+ }
+
+ private static void waitforLeaderToBeReady(RaftServer server, long timeout,
+ RaftGroup group)
+ throws IOException {
+ boolean ready;
+ long st = Time.monotonicNow();
+ do {
+ ready = server.getDivision(group.getGroupId()).getInfo().isLeaderReady();
+ if (!ready) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } while (!ready && Time.monotonicNow() - st < timeout);
- this.division =
- server.getDivision(server.getGroups().iterator().next().getGroupId());
+ if (!ready) {
+ throw new IOException(String
+ .format("Ratis group %s is not ready in %d ms", group.getGroupId(),
+ timeout));
+ }
}
- public static RaftServer.Builder newRaftServer(final String clusterId,
- final String scmId, final SCMHANodeDetails haDetails,
- final ConfigurationSource conf) throws IOException {
- final String scmNodeId = haDetails.getLocalNodeDetails().getNodeId();
+ private static RaftServer.Builder newRaftServer(final String scmId,
+ final ConfigurationSource conf) {
final SCMHAConfiguration haConf = conf.getObject(SCMHAConfiguration.class);
- SCMHAGroupBuilder haGrpBuilder = scmNodeId != null ?
- new SCMHAGroupBuilder(haDetails, clusterId, scmId) :
- new SCMHAGroupBuilder(haConf, conf, clusterId);
final RaftProperties serverProperties =
RatisUtil.newRaftProperties(haConf, conf);
- return RaftServer.newBuilder().setServerId(haGrpBuilder.getPeerId())
- .setGroup(haGrpBuilder.getRaftGroup()).setProperties(serverProperties)
- .setStateMachine(new SCMStateMachine());
+ return RaftServer.newBuilder().setServerId(RaftPeerId.getRaftPeerId(scmId))
+ .setProperties(serverProperties)
+ .setStateMachine(new SCMStateMachine(false));
}
@Override
public void start() throws IOException {
- division.getRaftServer().start();
+ server.start();
+ }
+
+ public RaftServer.Division getDivision() {
+ try {
+ return server
+ .getDivision(server.getGroups().iterator().next().getGroupId());
+ } catch (Exception e) {
+ LOG.warn("Failed to get RaftServerDivision", e);
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ public SCMStateMachine getStateMachine() {
+ return stateMachine;
}
@Override
public void registerStateMachineHandler(final RequestType handlerType,
final Object handler) {
- ((SCMStateMachine) division.getStateMachine())
- .registerHandler(handlerType, handler);
+ stateMachine.registerHandler(handlerType, handler);
}
@Override
public SCMRatisResponse submitRequest(SCMRatisRequest request)
throws IOException, ExecutionException, InterruptedException {
final RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder()
.setClientId(clientId)
Review comment:
Thanks for pointing this out. Addressed this.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583449673
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
Review comment:
This cannot happen. applyTransaction is executed in the log order and applTransaction completely sequential in SCM stateMachine.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583451454
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.Rule;
+
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
+
+/**
+ * Base class for Ozone Manager HA tests.
+ */
+public class TestStorageContainerManagerHA {
+
+ private MiniOzoneHAClusterImpl cluster = null;
+ private OzoneConfiguration conf;
+ private String clusterId;
+ private String scmId;
+ private String omServiceId;
+ private static int numOfOMs = 3;
+ private String scmServiceId;
+ // TODO: 3 servers will not work until SCM HA configs are considered in
+ // SCM HA Ratis server init
+ private static int numOfSCMs = 3;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omServiceId = "om-service-test1";
+ scmServiceId = "scm-service-test1";
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setScmId(scmId)
+ .setOMServiceId(omServiceId)
+ .setSCMServiceId(scmServiceId)
+ .setNumOfStorageContainerManagers(numOfSCMs)
+ .setNumOfOzoneManagers(numOfOMs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAllSCMAreRunning() throws Exception {
+ int count = 0;
+ List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
+ Assert.assertEquals(numOfSCMs, scms.size());
+ int peerSize = cluster.getStorageContainerManager().getScmHAManager()
+ .getRatisServer().getDivision().getGroup().getPeers().size();
+ for (StorageContainerManager scm : scms) {
+ if (scm.checkLeader()) {
+ count++;
+ }
+ Assert.assertTrue(peerSize == numOfSCMs);
+ }
+ Assert.assertEquals(1, count);
+ count = 0;
+ List<OzoneManager> oms = cluster.getOzoneManagersList();
+ Assert.assertEquals(numOfOMs, oms.size());
+ for (OzoneManager om : oms) {
+ if (om.isLeaderReady()) {
+ count++;
+ }
+ }
+ Assert.assertEquals(1, count);
+ testPutKey();
+ }
+
+ public void testPutKey() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ Instant testStartTime = Instant.now();
+ ObjectStore store =
+ OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length, STAND_ALONE, ONE,
+ new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
+ Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ is.close();
+ final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .setRefreshPipeline(true).build();
+ final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ final List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ final long containerID = keyLocationInfos.get(0).getContainerID();
+ for (int k = 0; k < numOfSCMs; k++) {
+ StorageContainerManager scm =
+ cluster.getStorageContainerManagers().get(k);
+ // flush to DB on each SCM
Review comment:
> That's why explicit take snapshot call is introduced so that, it first waits for all the appended log entries to complete apply on each node. It will ensure all are upto date.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583508127
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
+ TransactionInfo lastAppliedTrxInfo =
+ TransactionInfo.fromTermIndex(lastTermIndex);
+ if (transactionBuffer.getLatestTrxInfo().compareTo(lastAppliedTrxInfo)
+ < 0) {
+ transactionBuffer.updateLatestTrxInfo(
+ TransactionInfo.builder().setCurrentTerm(lastTermIndex.getTerm())
+ .setTransactionIndex(lastTermIndex.getIndex()).build());
+ transactionBuffer.setLatestSnapshot(
+ transactionBuffer.getLatestTrxInfo().toSnapshotInfo());
+ } else {
+ lastAppliedIndex =
+ transactionBuffer.getLatestTrxInfo().getTransactionIndex();
+ }
+
+ transactionBuffer.flush();
+ LOG.info("Current Snapshot Index {}, takeSnapshot took {} ms",
+ lastAppliedIndex, Time.monotonicNow() - startTime);
+ }
+ super.takeSnapshot();
+ return lastAppliedIndex;
+ }
+
+ @Override
+ public void initialize(
+ RaftServer server, RaftGroupId id, RaftStorage raftStorage)
+ throws IOException {
+ super.initialize(server, id, raftStorage);
+ storage.init(raftStorage);
+ loadSnapshot(storage.getLatestSnapshot());
+ }
+
+ private long loadSnapshot(SingleFileSnapshotInfo snapshot)
+ throws IOException {
+ if (snapshot == null) {
Review comment:
Yes, its not required.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583525849
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.Rule;
+
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
+
+/**
+ * Base class for Ozone Manager HA tests.
+ */
+public class TestStorageContainerManagerHA {
+
+ private MiniOzoneHAClusterImpl cluster = null;
+ private OzoneConfiguration conf;
+ private String clusterId;
+ private String scmId;
+ private String omServiceId;
+ private static int numOfOMs = 3;
+ private String scmServiceId;
+ // TODO: 3 servers will not work until SCM HA configs are considered in
+ // SCM HA Ratis server init
+ private static int numOfSCMs = 3;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omServiceId = "om-service-test1";
+ scmServiceId = "scm-service-test1";
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setScmId(scmId)
+ .setOMServiceId(omServiceId)
+ .setSCMServiceId(scmServiceId)
+ .setNumOfStorageContainerManagers(numOfSCMs)
+ .setNumOfOzoneManagers(numOfOMs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAllSCMAreRunning() throws Exception {
+ int count = 0;
+ List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
+ Assert.assertEquals(numOfSCMs, scms.size());
+ int peerSize = cluster.getStorageContainerManager().getScmHAManager()
+ .getRatisServer().getDivision().getGroup().getPeers().size();
+ for (StorageContainerManager scm : scms) {
+ if (scm.checkLeader()) {
+ count++;
+ }
+ Assert.assertTrue(peerSize == numOfSCMs);
+ }
+ Assert.assertEquals(1, count);
+ count = 0;
+ List<OzoneManager> oms = cluster.getOzoneManagersList();
+ Assert.assertEquals(numOfOMs, oms.size());
+ for (OzoneManager om : oms) {
+ if (om.isLeaderReady()) {
+ count++;
+ }
+ }
+ Assert.assertEquals(1, count);
+ testPutKey();
+ }
+
+ public void testPutKey() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ Instant testStartTime = Instant.now();
+ ObjectStore store =
+ OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length, STAND_ALONE, ONE,
+ new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
+ Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ is.close();
+ final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .setRefreshPipeline(true).build();
+ final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ final List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ final long containerID = keyLocationInfos.get(0).getContainerID();
+ for (int k = 0; k < numOfSCMs; k++) {
+ StorageContainerManager scm =
+ cluster.getStorageContainerManagers().get(k);
+ // flush to DB on each SCM
Review comment:
Addressed in the recent. It is waiting for follower catch up before flush.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583570843
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
##########
@@ -708,14 +708,14 @@ public static boolean scmBootstrap(OzoneConfiguration conf)
}
// The node here will try to fetch the cluster id from any of existing
// running SCM instances.
- // TODO: need to avoid failover to local SCM Node here
- final ScmInfo scmInfo = HAUtils.getScmInfo(conf);
Review comment:
Done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bharatviswa504 commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bharatviswa504 commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583504159
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
+ TransactionInfo lastAppliedTrxInfo =
+ TransactionInfo.fromTermIndex(lastTermIndex);
+ if (transactionBuffer.getLatestTrxInfo().compareTo(lastAppliedTrxInfo)
+ < 0) {
+ transactionBuffer.updateLatestTrxInfo(
+ TransactionInfo.builder().setCurrentTerm(lastTermIndex.getTerm())
+ .setTransactionIndex(lastTermIndex.getIndex()).build());
+ transactionBuffer.setLatestSnapshot(
+ transactionBuffer.getLatestTrxInfo().toSnapshotInfo());
+ } else {
+ lastAppliedIndex =
+ transactionBuffer.getLatestTrxInfo().getTransactionIndex();
+ }
+
+ transactionBuffer.flush();
+ LOG.info("Current Snapshot Index {}, takeSnapshot took {} ms",
+ lastAppliedIndex, Time.monotonicNow() - startTime);
+ }
+ super.takeSnapshot();
+ return lastAppliedIndex;
+ }
+
+ @Override
+ public void initialize(
+ RaftServer server, RaftGroupId id, RaftStorage raftStorage)
+ throws IOException {
+ super.initialize(server, id, raftStorage);
+ storage.init(raftStorage);
+ loadSnapshot(storage.getLatestSnapshot());
+ }
+
+ private long loadSnapshot(SingleFileSnapshotInfo snapshot)
+ throws IOException {
+ if (snapshot == null) {
Review comment:
This is completely removed now. I don't see this fixed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583448845
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
+ TransactionInfo lastAppliedTrxInfo =
+ TransactionInfo.fromTermIndex(lastTermIndex);
+ if (transactionBuffer.getLatestTrxInfo().compareTo(lastAppliedTrxInfo)
+ < 0) {
+ transactionBuffer.updateLatestTrxInfo(
+ TransactionInfo.builder().setCurrentTerm(lastTermIndex.getTerm())
+ .setTransactionIndex(lastTermIndex.getIndex()).build());
+ transactionBuffer.setLatestSnapshot(
+ transactionBuffer.getLatestTrxInfo().toSnapshotInfo());
+ } else {
+ lastAppliedIndex =
+ transactionBuffer.getLatestTrxInfo().getTransactionIndex();
+ }
+
+ transactionBuffer.flush();
+ LOG.info("Current Snapshot Index {}, takeSnapshot took {} ms",
+ lastAppliedIndex, Time.monotonicNow() - startTime);
+ }
+ super.takeSnapshot();
Review comment:
yes, to ensure if there are any updates to sm directory by the base stateMachine impelementation.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant merged pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant merged pull request #1953:
URL: https://github.com/apache/ozone/pull/1953
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#issuecomment-787475756
> Overall LGTM. One comment regarding usage of clientID and callID usage need to be set from Server API's that looks missing.
> And also getting localNodeID need to be fetched, as setNodeSpecificConfigKeys does not ozone.scm.node.id.
There is only one central place of submitting ratis requests and there is only client Id which will be same for all requests submitted to the server on each node. I don't think , we need to get the clientId from server here as there is only one client.
Same goes for "callId" as well.
However, this change is not related to the patch. We can discuss it further and make change accordingly, in a different jira.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583525465
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
##########
@@ -0,0 +1,738 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.safemode.HealthyPipelineSafeModeRule;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.recon.ReconServer;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.BindException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * MiniOzoneHAClusterImpl creates a complete in-process Ozone cluster
+ * with OM HA and SCM HA suitable for running tests.
+ * The cluster consists of a set of
+ * OzoneManagers, StorageContainerManagers and multiple DataNodes.
+ */
+public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MiniOzoneHAClusterImpl.class);
+
+ private final OMHAService omhaService;
+ private final SCMHAService scmhaService;
+ private static ObjectStore store = null;
+
+ private int waitForClusterToBeReadyTimeout = 120000; // 2 min
+
+ private static final Random RANDOM = new Random();
+ private static final int RATIS_RPC_TIMEOUT = 1000; // 1 second
+ public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
+
+ /**
+ * Creates a new MiniOzoneCluster.
+ *
+ * @throws IOException if there is an I/O error
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> activeOMList,
+ List<OzoneManager> inactiveOMList,
+ List<StorageContainerManager> activeSCMList,
+ List<StorageContainerManager> inactiveSCMList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId,
+ ReconServer reconServer) {
+ super(conf, hddsDatanodes, reconServer);
+ omhaService =
+ new OMHAService(activeOMList, inactiveOMList, omServiceId);
+ scmhaService =
+ new SCMHAService(activeSCMList, inactiveSCMList, scmServiceId);
+ }
+
+ /**
+ * Creates a new MiniOzoneCluster with all OMs active.
+ * This is used by MiniOzoneChaosCluster.
+ */
+ protected MiniOzoneHAClusterImpl(
+ OzoneConfiguration conf,
+ List<OzoneManager> omList,
+ List<StorageContainerManager> scmList,
+ List<HddsDatanodeService> hddsDatanodes,
+ String omServiceId,
+ String scmServiceId) {
+ this(conf, omList, null, scmList, null, hddsDatanodes,
+ omServiceId, scmServiceId, null);
+ }
+
+ @Override
+ public String getOMServiceId() {
+ return omhaService.getServiceId();
+ }
+
+ @Override
+ public String getSCMServiceId() {
+ return scmhaService.getServiceId();
+ }
+
+ /**
+ * Returns the first OzoneManager from the list.
+ * @return
+ */
+ @Override
+ public OzoneManager getOzoneManager() {
+ return this.omhaService.getServices().get(0);
+ }
+
+ @Override
+ public OzoneClient getRpcClient() throws IOException {
+ String omServiceId = omhaService.getServiceId();
+ if (omServiceId == null) {
+ // Non-HA cluster.
+ return OzoneClientFactory.getRpcClient(getConf());
+ } else {
+ // HA cluster
+ return OzoneClientFactory.getRpcClient(omServiceId, getConf());
+ }
+ }
+
+ public boolean isOMActive(String omNodeId) {
+ return omhaService.isServiceActive(omNodeId);
+ }
+
+ public OzoneManager getOzoneManager(int index) {
+ return this.omhaService.getServiceByIndex(index);
+ }
+
+ public OzoneManager getOzoneManager(String omNodeId) {
+ return this.omhaService.getServiceById(omNodeId);
+ }
+
+ public List<OzoneManager> getOzoneManagersList() {
+ return omhaService.getServices();
+ }
+
+ /**
+ * Get OzoneManager leader object.
+ * @return OzoneManager object, null if there isn't one or more than one
+ */
+ public OzoneManager getOMLeader() {
+ OzoneManager res = null;
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager.isLeaderReady()) {
+ if (res != null) {
+ // Found more than one leader
+ // Return null, expect the caller to retry in a while
+ return null;
+ }
+ // Found a leader
+ res = ozoneManager;
+ }
+ }
+ return res;
+ }
+
+ /**
+ * Start a previously inactive OM.
+ */
+ public void startInactiveOM(String omNodeID) throws IOException {
+ omhaService.startInactiveService(omNodeID, OzoneManager::start);
+ }
+
+ @Override
+ public void restartOzoneManager() throws IOException {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ ozoneManager.stop();
+ ozoneManager.restart();
+ }
+ }
+
+ public void shutdownOzoneManager(OzoneManager ozoneManager) {
+ LOG.info("Shutting down OzoneManager " + ozoneManager.getOMNodeId());
+
+ ozoneManager.stop();
+ }
+
+ public void restartOzoneManager(OzoneManager ozoneManager, boolean waitForOM)
+ throws IOException, TimeoutException, InterruptedException {
+ LOG.info("Restarting OzoneManager " + ozoneManager.getOMNodeId());
+ ozoneManager.restart();
+
+ if (waitForOM) {
+ GenericTestUtils.waitFor(ozoneManager::isRunning,
+ 1000, waitForClusterToBeReadyTimeout);
+ }
+ }
+
+ public String getClusterId() throws IOException {
+ return scmhaService.getServices().get(0)
+ .getClientProtocolServer().getScmInfo().getClusterId();
+ }
+
+ public StorageContainerManager getActiveSCM() {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return scm;
+ }
+ }
+ return null;
+ }
+
+ public void waitForSCMToBeReady()
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(() -> {
+ for (StorageContainerManager scm : scmhaService.getServices()) {
+ if (scm.checkLeader()) {
+ return true;
+ }
+ }
+ return false;
+ }, 1000, waitForClusterToBeReadyTimeout);
+ }
+
+ @Override
+ public void stop() {
+ for (OzoneManager ozoneManager : this.omhaService.getServices()) {
+ if (ozoneManager != null) {
+ LOG.info("Stopping the OzoneManager {}", ozoneManager.getOMNodeId());
+ ozoneManager.stop();
+ ozoneManager.join();
+ }
+ }
+
+ for (StorageContainerManager scm : this.scmhaService.getServices()) {
+ if (scm != null) {
+ LOG.info("Stopping the StorageContainerManager {}", scm.getScmId());
+ scm.stop();
+ scm.join();
+ }
+ }
+ super.stop();
+ }
+
+ public void stopOzoneManager(int index) {
+ omhaService.getServices().get(index).stop();
+ omhaService.getServices().get(index).join();
+ }
+
+ public void stopOzoneManager(String omNodeId) {
+ omhaService.getServiceById(omNodeId).stop();
+ omhaService.getServiceById(omNodeId).join();
+ }
+
+ /**
+ * Builder for configuring the MiniOzoneCluster to run.
+ */
+ public static class Builder extends MiniOzoneClusterImpl.Builder {
+
+ private static final String OM_NODE_ID_PREFIX = "omNode-";
+ private List<OzoneManager> activeOMs = new ArrayList<>();
+ private List<OzoneManager> inactiveOMs = new ArrayList<>();
+
+ private static final String SCM_NODE_ID_PREFIX = "scmNode-";
+ private List<StorageContainerManager> activeSCMs = new ArrayList<>();
+ private List<StorageContainerManager> inactiveSCMs = new ArrayList<>();
+
+ /**
+ * Creates a new Builder.
+ *
+ * @param conf configuration
+ */
+ public Builder(OzoneConfiguration conf) {
+ super(conf);
+ }
+
+ public List<OzoneManager> getActiveOMs() {
+ return activeOMs;
+ }
+
+ public List<OzoneManager> getInactiveOMs() {
+ return inactiveOMs;
+ }
+
+ @Override
+ public MiniOzoneCluster build() throws IOException {
+ if (numOfActiveOMs > numOfOMs) {
+ throw new IllegalArgumentException("Number of active OMs cannot be " +
+ "more than the total number of OMs");
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveOMs == ACTIVE_OMS_NOT_SET) {
+ numOfActiveOMs = numOfOMs;
+ }
+
+ // If num of ActiveOMs is not set, set it to numOfOMs.
+ if (numOfActiveSCMs == ACTIVE_SCMS_NOT_SET) {
+ numOfActiveSCMs = numOfSCMs;
+ }
+
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ initializeConfiguration();
+ initOMRatisConf();
+ StorageContainerManager scm;
+ ReconServer reconServer = null;
+ try {
+ createSCMService();
+ createOMService();
+ if (includeRecon) {
+ configureRecon();
+ reconServer = new ReconServer();
+ reconServer.execute(new String[] {});
+ }
+ } catch (AuthenticationException ex) {
+ throw new IOException("Unable to build MiniOzoneCluster. ", ex);
+ }
+
+ final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(
+ activeSCMs, reconServer);
+
+ MiniOzoneHAClusterImpl cluster = new MiniOzoneHAClusterImpl(conf,
+ activeOMs, inactiveOMs, activeSCMs, inactiveSCMs,
+ hddsDatanodes, omServiceId, scmServiceId, reconServer);
+
+ if (startDataNodes) {
+ cluster.startHddsDatanodes();
+ }
+ return cluster;
+ }
+
+ protected void initOMRatisConf() {
+ conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
+ conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers);
+
+ // If test change the following config values we will respect,
+ // otherwise we will set lower timeout values.
+ long defaultDuration = OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT
+ .getDuration();
+ long curRatisRpcTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ defaultDuration == curRatisRpcTimeout ?
+ RATIS_RPC_TIMEOUT : curRatisRpcTimeout, TimeUnit.MILLISECONDS);
+
+ long defaultNodeFailureTimeout =
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getDuration();
+ long curNodeFailureTimeout = conf.getTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ defaultNodeFailureTimeout,
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT.
+ getUnit());
+ conf.setTimeDuration(
+ OMConfigKeys.OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_KEY,
+ curNodeFailureTimeout == defaultNodeFailureTimeout ?
+ NODE_FAILURE_TIMEOUT : curNodeFailureTimeout,
+ TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<OzoneManager> createOMService() throws IOException,
+ AuthenticationException {
+
+ List<OzoneManager> omList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 10000;
+
+ while (true) {
+ try {
+ basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ initOMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfOMs; i++) {
+ // Set nodeId
+ String nodeId = OM_NODE_ID_PREFIX + i;
+ OzoneConfiguration config = new OzoneConfiguration(conf);
+ config.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeId);
+ // Set the OM http(s) address to null so that the cluster picks
+ // up the address set with service ID and node ID in initHAConfig
+ config.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "");
+ config.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, "");
+
+ // Set metadata/DB dir base path
+ String metaDirPath = path + "/" + nodeId;
+ config.set(OZONE_METADATA_DIRS, metaDirPath);
+ // OMStorage omStore = new OMStorage(config);
+ // initializeOmStorage(omStore);
+ OzoneManager.omInit(config);
+ OzoneManager om = OzoneManager.createOm(config);
+ if (certClient != null) {
+ om.setCertClient(certClient);
+ }
+ omList.add(om);
+
+ if (i <= numOfActiveOMs) {
+ om.start();
+ activeOMs.add(om);
+ LOG.info("Started OzoneManager RPC server at {}",
+ om.getOmRpcServerAddr());
+ } else {
+ inactiveOMs.add(om);
+ LOG.info("Intialized OzoneManager at {}. This OM is currently "
+ + "inactive (not running).", om.getOmRpcServerAddr());
+ }
+ }
+
+ // Set default OM address to point to the first OM. Clients would
+ // try connecting to this address by default
+ conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
+ NetUtils.getHostPortString(omList.get(0).getOmRpcServerAddr()));
+
+ break;
+ } catch (BindException e) {
+ for (OzoneManager om : omList) {
+ om.stop();
+ om.join();
+ LOG.info("Stopping OzoneManager server at {}",
+ om.getOmRpcServerAddr());
+ }
+ omList.clear();
+ ++retryCount;
+ LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
+ retryCount);
+ }
+ }
+ return omList;
+ }
+
+ /**
+ * Start OM service with multiple OMs.
+ */
+ protected List<StorageContainerManager> createSCMService()
+ throws IOException, AuthenticationException {
+ List<StorageContainerManager> scmList = Lists.newArrayList();
+
+ int retryCount = 0;
+ int basePort = 12000;
+
+ while (true) {
+ try {
+ basePort = 12000 + RANDOM.nextInt(1000) * 4;
+ initSCMHAConfig(basePort);
+
+ for (int i = 1; i<= numOfSCMs; i++) {
+ // Set nodeId
+ String nodeId = SCM_NODE_ID_PREFIX + i;
+ String metaDirPath = path + "/" + nodeId;
+ OzoneConfiguration scmConfig = new OzoneConfiguration(conf);
+ scmConfig.set(OZONE_METADATA_DIRS, metaDirPath);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_NODE_ID_KEY, nodeId);
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "");
+ scmConfig.set(ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_KEY, "");
+ scmConfig.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+
+ // TODO: set SCM HA configs
+
+ configureSCM();
+ if (i == 1) {
+ StorageContainerManager.scmInit(scmConfig, clusterId);
+ } else {
+ StorageContainerManager.scmBootstrap(scmConfig);
+ }
+ StorageContainerManager scm = TestUtils.getScmSimple(scmConfig);
+ HealthyPipelineSafeModeRule rule =
+ scm.getScmSafeModeManager().getHealthyPipelineSafeModeRule();
+ if (rule != null) {
+ // Set threshold to wait for safe mode exit -
+ // this is needed since a pipeline is marked open only after
+ // leader election.
+ rule.setHealthyPipelineThresholdCount(numOfDatanodes / 3);
+ }
+ scmList.add(scm);
+
+ if (i <= numOfActiveSCMs) {
+ scm.start();
+ activeSCMs.add(scm);
+ LOG.info("Started SCM RPC server at {}",
+ scm.getClientProtocolServer());
+ } else {
+ inactiveSCMs.add(scm);
+ LOG.info("Intialized SCM at {}. This SCM is currently "
+ + "inactive (not running).", scm.getClientProtocolServer());
+ }
+ }
+
+
+ break;
+ } catch (BindException e) {
+ for (StorageContainerManager scm : scmList) {
+ scm.stop();
+ scm.join();
+ LOG.info("Stopping StorageContainerManager server at {}",
+ scm.getClientProtocolServer());
+ }
+ scmList.clear();
+ ++retryCount;
+ LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
+ retryCount);
+ }
+ }
+ return scmList;
+ }
+
+ protected void configureSCM() {
Review comment:
It will be called from super class.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583563075
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
+ server.close();
+ }
+
+ private static void waitforLeaderToBeReady(RaftServer server, long timeout,
+ RaftGroup group)
+ throws IOException {
+ boolean ready;
+ long st = Time.monotonicNow();
+ do {
+ ready = server.getDivision(group.getGroupId()).getInfo().isLeaderReady();
+ if (!ready) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } while (!ready && Time.monotonicNow() - st < timeout);
- this.division =
- server.getDivision(server.getGroups().iterator().next().getGroupId());
+ if (!ready) {
+ throw new IOException(String
+ .format("Ratis group %s is not ready in %d ms", group.getGroupId(),
+ timeout));
+ }
}
- public static RaftServer.Builder newRaftServer(final String clusterId,
- final String scmId, final SCMHANodeDetails haDetails,
- final ConfigurationSource conf) throws IOException {
- final String scmNodeId = haDetails.getLocalNodeDetails().getNodeId();
+ private static RaftServer.Builder newRaftServer(final String scmId,
+ final ConfigurationSource conf) {
final SCMHAConfiguration haConf = conf.getObject(SCMHAConfiguration.class);
- SCMHAGroupBuilder haGrpBuilder = scmNodeId != null ?
- new SCMHAGroupBuilder(haDetails, clusterId, scmId) :
- new SCMHAGroupBuilder(haConf, conf, clusterId);
final RaftProperties serverProperties =
RatisUtil.newRaftProperties(haConf, conf);
- return RaftServer.newBuilder().setServerId(haGrpBuilder.getPeerId())
- .setGroup(haGrpBuilder.getRaftGroup()).setProperties(serverProperties)
- .setStateMachine(new SCMStateMachine());
+ return RaftServer.newBuilder().setServerId(RaftPeerId.getRaftPeerId(scmId))
+ .setProperties(serverProperties)
+ .setStateMachine(new SCMStateMachine(false));
}
@Override
public void start() throws IOException {
- division.getRaftServer().start();
+ server.start();
+ }
+
+ public RaftServer.Division getDivision() {
+ try {
+ return server
+ .getDivision(server.getGroups().iterator().next().getGroupId());
+ } catch (Exception e) {
+ LOG.warn("Failed to get RaftServerDivision", e);
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ public SCMStateMachine getStateMachine() {
+ return stateMachine;
}
@Override
public void registerStateMachineHandler(final RequestType handlerType,
final Object handler) {
- ((SCMStateMachine) division.getStateMachine())
- .registerHandler(handlerType, handler);
+ stateMachine.registerHandler(handlerType, handler);
}
@Override
public SCMRatisResponse submitRequest(SCMRatisRequest request)
throws IOException, ExecutionException, InterruptedException {
final RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder()
.setClientId(clientId)
- .setServerId(division.getId())
- .setGroupId(division.getGroup().getGroupId())
+ .setServerId(getDivision().getId())
Review comment:
Division info can change depending upon server state. Its safe to use this via Ratis. Its a cached object away inside ratisServerImpl.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583557394
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
Review comment:
I am adding TODO to make the timeout as well as sleep interval configurable. I think we should make it dependent on Ratis Leader election timeout.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583486288
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.Rule;
+
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
+
+/**
+ * Base class for Ozone Manager HA tests.
+ */
+public class TestStorageContainerManagerHA {
+
+ private MiniOzoneHAClusterImpl cluster = null;
+ private OzoneConfiguration conf;
+ private String clusterId;
+ private String scmId;
+ private String omServiceId;
+ private static int numOfOMs = 3;
+ private String scmServiceId;
+ // TODO: 3 servers will not work until SCM HA configs are considered in
+ // SCM HA Ratis server init
+ private static int numOfSCMs = 3;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omServiceId = "om-service-test1";
+ scmServiceId = "scm-service-test1";
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setScmId(scmId)
+ .setOMServiceId(omServiceId)
+ .setSCMServiceId(scmServiceId)
+ .setNumOfStorageContainerManagers(numOfSCMs)
+ .setNumOfOzoneManagers(numOfOMs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAllSCMAreRunning() throws Exception {
+ int count = 0;
+ List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
+ Assert.assertEquals(numOfSCMs, scms.size());
+ int peerSize = cluster.getStorageContainerManager().getScmHAManager()
+ .getRatisServer().getDivision().getGroup().getPeers().size();
+ for (StorageContainerManager scm : scms) {
+ if (scm.checkLeader()) {
+ count++;
+ }
+ Assert.assertTrue(peerSize == numOfSCMs);
+ }
+ Assert.assertEquals(1, count);
+ count = 0;
+ List<OzoneManager> oms = cluster.getOzoneManagersList();
+ Assert.assertEquals(numOfOMs, oms.size());
+ for (OzoneManager om : oms) {
+ if (om.isLeaderReady()) {
+ count++;
+ }
+ }
+ Assert.assertEquals(1, count);
+ testPutKey();
+ }
+
+ public void testPutKey() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ Instant testStartTime = Instant.now();
+ ObjectStore store =
+ OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length, STAND_ALONE, ONE,
+ new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
+ Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ is.close();
+ final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .setRefreshPipeline(true).build();
+ final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ final List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ final long containerID = keyLocationInfos.get(0).getContainerID();
+ for (int k = 0; k < numOfSCMs; k++) {
+ StorageContainerManager scm =
+ cluster.getStorageContainerManagers().get(k);
+ // flush to DB on each SCM
Review comment:
Snapshot ensures whatever is in the log is applied.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583554571
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
Review comment:
yes..
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583508539
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
+ TransactionInfo lastAppliedTrxInfo =
+ TransactionInfo.fromTermIndex(lastTermIndex);
+ if (transactionBuffer.getLatestTrxInfo().compareTo(lastAppliedTrxInfo)
+ < 0) {
+ transactionBuffer.updateLatestTrxInfo(
+ TransactionInfo.builder().setCurrentTerm(lastTermIndex.getTerm())
+ .setTransactionIndex(lastTermIndex.getIndex()).build());
+ transactionBuffer.setLatestSnapshot(
+ transactionBuffer.getLatestTrxInfo().toSnapshotInfo());
+ } else {
+ lastAppliedIndex =
+ transactionBuffer.getLatestTrxInfo().getTransactionIndex();
+ }
+
+ transactionBuffer.flush();
+ LOG.info("Current Snapshot Index {}, takeSnapshot took {} ms",
+ lastAppliedIndex, Time.monotonicNow() - startTime);
+ }
+ super.takeSnapshot();
+ return lastAppliedIndex;
+ }
+
+ @Override
+ public void initialize(
+ RaftServer server, RaftGroupId id, RaftStorage raftStorage)
+ throws IOException {
+ super.initialize(server, id, raftStorage);
+ storage.init(raftStorage);
+ loadSnapshot(storage.getLatestSnapshot());
+ }
+
+ private long loadSnapshot(SingleFileSnapshotInfo snapshot)
+ throws IOException {
+ if (snapshot == null) {
Review comment:
for the latest snapshot, getLatestSnapshot is already overriden in SCMStateMachine.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583571142
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
+ server.close();
+ }
+
+ private static void waitforLeaderToBeReady(RaftServer server, long timeout,
+ RaftGroup group)
+ throws IOException {
+ boolean ready;
+ long st = Time.monotonicNow();
+ do {
+ ready = server.getDivision(group.getGroupId()).getInfo().isLeaderReady();
+ if (!ready) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } while (!ready && Time.monotonicNow() - st < timeout);
- this.division =
- server.getDivision(server.getGroups().iterator().next().getGroupId());
+ if (!ready) {
+ throw new IOException(String
+ .format("Ratis group %s is not ready in %d ms", group.getGroupId(),
+ timeout));
+ }
}
- public static RaftServer.Builder newRaftServer(final String clusterId,
- final String scmId, final SCMHANodeDetails haDetails,
- final ConfigurationSource conf) throws IOException {
- final String scmNodeId = haDetails.getLocalNodeDetails().getNodeId();
+ private static RaftServer.Builder newRaftServer(final String scmId,
+ final ConfigurationSource conf) {
final SCMHAConfiguration haConf = conf.getObject(SCMHAConfiguration.class);
- SCMHAGroupBuilder haGrpBuilder = scmNodeId != null ?
- new SCMHAGroupBuilder(haDetails, clusterId, scmId) :
- new SCMHAGroupBuilder(haConf, conf, clusterId);
final RaftProperties serverProperties =
RatisUtil.newRaftProperties(haConf, conf);
- return RaftServer.newBuilder().setServerId(haGrpBuilder.getPeerId())
- .setGroup(haGrpBuilder.getRaftGroup()).setProperties(serverProperties)
- .setStateMachine(new SCMStateMachine());
+ return RaftServer.newBuilder().setServerId(RaftPeerId.getRaftPeerId(scmId))
+ .setProperties(serverProperties)
+ .setStateMachine(new SCMStateMachine(false));
}
@Override
public void start() throws IOException {
- division.getRaftServer().start();
+ server.start();
+ }
+
+ public RaftServer.Division getDivision() {
+ try {
+ return server
+ .getDivision(server.getGroups().iterator().next().getGroupId());
+ } catch (Exception e) {
+ LOG.warn("Failed to get RaftServerDivision", e);
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ public SCMStateMachine getStateMachine() {
+ return stateMachine;
}
@Override
public void registerStateMachineHandler(final RequestType handlerType,
final Object handler) {
- ((SCMStateMachine) division.getStateMachine())
- .registerHandler(handlerType, handler);
+ stateMachine.registerHandler(handlerType, handler);
}
@Override
public SCMRatisResponse submitRequest(SCMRatisRequest request)
throws IOException, ExecutionException, InterruptedException {
final RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder()
.setClientId(clientId)
Review comment:
Thanks for pointing this out. Addressed this.
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
+ server.close();
+ }
+
+ private static void waitforLeaderToBeReady(RaftServer server, long timeout,
+ RaftGroup group)
+ throws IOException {
+ boolean ready;
+ long st = Time.monotonicNow();
+ do {
+ ready = server.getDivision(group.getGroupId()).getInfo().isLeaderReady();
+ if (!ready) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } while (!ready && Time.monotonicNow() - st < timeout);
- this.division =
- server.getDivision(server.getGroups().iterator().next().getGroupId());
+ if (!ready) {
+ throw new IOException(String
+ .format("Ratis group %s is not ready in %d ms", group.getGroupId(),
+ timeout));
+ }
}
- public static RaftServer.Builder newRaftServer(final String clusterId,
- final String scmId, final SCMHANodeDetails haDetails,
- final ConfigurationSource conf) throws IOException {
- final String scmNodeId = haDetails.getLocalNodeDetails().getNodeId();
+ private static RaftServer.Builder newRaftServer(final String scmId,
+ final ConfigurationSource conf) {
final SCMHAConfiguration haConf = conf.getObject(SCMHAConfiguration.class);
- SCMHAGroupBuilder haGrpBuilder = scmNodeId != null ?
- new SCMHAGroupBuilder(haDetails, clusterId, scmId) :
- new SCMHAGroupBuilder(haConf, conf, clusterId);
final RaftProperties serverProperties =
RatisUtil.newRaftProperties(haConf, conf);
- return RaftServer.newBuilder().setServerId(haGrpBuilder.getPeerId())
- .setGroup(haGrpBuilder.getRaftGroup()).setProperties(serverProperties)
- .setStateMachine(new SCMStateMachine());
+ return RaftServer.newBuilder().setServerId(RaftPeerId.getRaftPeerId(scmId))
+ .setProperties(serverProperties)
+ .setStateMachine(new SCMStateMachine(false));
}
@Override
public void start() throws IOException {
- division.getRaftServer().start();
+ server.start();
+ }
+
+ public RaftServer.Division getDivision() {
+ try {
+ return server
+ .getDivision(server.getGroups().iterator().next().getGroupId());
+ } catch (Exception e) {
+ LOG.warn("Failed to get RaftServerDivision", e);
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ public SCMStateMachine getStateMachine() {
+ return stateMachine;
}
@Override
public void registerStateMachineHandler(final RequestType handlerType,
final Object handler) {
- ((SCMStateMachine) division.getStateMachine())
- .registerHandler(handlerType, handler);
+ stateMachine.registerHandler(handlerType, handler);
}
@Override
public SCMRatisResponse submitRequest(SCMRatisRequest request)
throws IOException, ExecutionException, InterruptedException {
final RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder()
.setClientId(clientId)
- .setServerId(division.getId())
- .setGroupId(division.getGroup().getGroupId())
+ .setServerId(getDivision().getId())
Review comment:
Done.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bharatviswa504 commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bharatviswa504 commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583452686
##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.Rule;
+
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
+import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE;
+
+/**
+ * Base class for Ozone Manager HA tests.
+ */
+public class TestStorageContainerManagerHA {
+
+ private MiniOzoneHAClusterImpl cluster = null;
+ private OzoneConfiguration conf;
+ private String clusterId;
+ private String scmId;
+ private String omServiceId;
+ private static int numOfOMs = 3;
+ private String scmServiceId;
+ // TODO: 3 servers will not work until SCM HA configs are considered in
+ // SCM HA Ratis server init
+ private static int numOfSCMs = 3;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(300_000);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omServiceId = "om-service-test1";
+ scmServiceId = "scm-service-test1";
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setClusterId(clusterId)
+ .setScmId(scmId)
+ .setOMServiceId(omServiceId)
+ .setSCMServiceId(scmServiceId)
+ .setNumOfStorageContainerManagers(numOfSCMs)
+ .setNumOfOzoneManagers(numOfOMs)
+ .build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testAllSCMAreRunning() throws Exception {
+ int count = 0;
+ List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
+ Assert.assertEquals(numOfSCMs, scms.size());
+ int peerSize = cluster.getStorageContainerManager().getScmHAManager()
+ .getRatisServer().getDivision().getGroup().getPeers().size();
+ for (StorageContainerManager scm : scms) {
+ if (scm.checkLeader()) {
+ count++;
+ }
+ Assert.assertTrue(peerSize == numOfSCMs);
+ }
+ Assert.assertEquals(1, count);
+ count = 0;
+ List<OzoneManager> oms = cluster.getOzoneManagersList();
+ Assert.assertEquals(numOfOMs, oms.size());
+ for (OzoneManager om : oms) {
+ if (om.isLeaderReady()) {
+ count++;
+ }
+ }
+ Assert.assertEquals(1, count);
+ testPutKey();
+ }
+
+ public void testPutKey() throws IOException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ Instant testStartTime = Instant.now();
+ ObjectStore store =
+ OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
+ String value = "sample value";
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket
+ .createKey(keyName, value.getBytes(UTF_8).length, STAND_ALONE, ONE,
+ new HashMap<>());
+ out.write(value.getBytes(UTF_8));
+ out.close();
+ OzoneKey key = bucket.getKey(keyName);
+ Assert.assertEquals(keyName, key.getName());
+ OzoneInputStream is = bucket.readKey(keyName);
+ byte[] fileContent = new byte[value.getBytes(UTF_8).length];
+ is.read(fileContent);
+ Assert.assertEquals(value, new String(fileContent, UTF_8));
+ Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
+ Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ is.close();
+ final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .setRefreshPipeline(true).build();
+ final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ final List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ final long containerID = keyLocationInfos.get(0).getContainerID();
+ for (int k = 0; k < numOfSCMs; k++) {
+ StorageContainerManager scm =
+ cluster.getStorageContainerManagers().get(k);
+ // flush to DB on each SCM
Review comment:
This is an explicit call that is being performed here, lets say 3rd SCM still not got in to applyphase for that transaction, then 3rd SCM will still not have right?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583478313
##########
File path: hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
##########
@@ -77,6 +78,11 @@
*/
ScmInfo getScmInfo() throws IOException;
+ /**
+ * Request to add SCM instance to HA group.
+ */
+ boolean addSCM(AddSCMRequest request) throws IOException;
Review comment:
I agree. Let's discuss it further.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] GlenGeng commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
GlenGeng commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r581747289
##########
File path: hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
##########
@@ -77,6 +78,11 @@
*/
ScmInfo getScmInfo() throws IOException;
+ /**
+ * Request to add SCM instance to HA group.
+ */
+ boolean addSCM(AddSCMRequest request) throws IOException;
Review comment:
How about move to `StorageContainerLocationProtocol`, where all the admin operations locate.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bharatviswa504 commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bharatviswa504 commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r584261881
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
+ server.close();
+ }
+
+ private static void waitforLeaderToBeReady(RaftServer server, long timeout,
+ RaftGroup group)
+ throws IOException {
+ boolean ready;
+ long st = Time.monotonicNow();
+ do {
+ ready = server.getDivision(group.getGroupId()).getInfo().isLeaderReady();
+ if (!ready) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } while (!ready && Time.monotonicNow() - st < timeout);
- this.division =
- server.getDivision(server.getGroups().iterator().next().getGroupId());
+ if (!ready) {
+ throw new IOException(String
+ .format("Ratis group %s is not ready in %d ms", group.getGroupId(),
+ timeout));
+ }
}
- public static RaftServer.Builder newRaftServer(final String clusterId,
- final String scmId, final SCMHANodeDetails haDetails,
- final ConfigurationSource conf) throws IOException {
- final String scmNodeId = haDetails.getLocalNodeDetails().getNodeId();
+ private static RaftServer.Builder newRaftServer(final String scmId,
+ final ConfigurationSource conf) {
final SCMHAConfiguration haConf = conf.getObject(SCMHAConfiguration.class);
- SCMHAGroupBuilder haGrpBuilder = scmNodeId != null ?
- new SCMHAGroupBuilder(haDetails, clusterId, scmId) :
- new SCMHAGroupBuilder(haConf, conf, clusterId);
final RaftProperties serverProperties =
RatisUtil.newRaftProperties(haConf, conf);
- return RaftServer.newBuilder().setServerId(haGrpBuilder.getPeerId())
- .setGroup(haGrpBuilder.getRaftGroup()).setProperties(serverProperties)
- .setStateMachine(new SCMStateMachine());
+ return RaftServer.newBuilder().setServerId(RaftPeerId.getRaftPeerId(scmId))
+ .setProperties(serverProperties)
+ .setStateMachine(new SCMStateMachine(false));
}
@Override
public void start() throws IOException {
- division.getRaftServer().start();
+ server.start();
+ }
+
+ public RaftServer.Division getDivision() {
+ try {
+ return server
+ .getDivision(server.getGroups().iterator().next().getGroupId());
+ } catch (Exception e) {
+ LOG.warn("Failed to get RaftServerDivision", e);
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ public SCMStateMachine getStateMachine() {
+ return stateMachine;
}
@Override
public void registerStateMachineHandler(final RequestType handlerType,
final Object handler) {
- ((SCMStateMachine) division.getStateMachine())
- .registerHandler(handlerType, handler);
+ stateMachine.registerHandler(handlerType, handler);
}
@Override
public SCMRatisResponse submitRequest(SCMRatisRequest request)
throws IOException, ExecutionException, InterruptedException {
final RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder()
.setClientId(clientId)
Review comment:
I don't see this being addressed?
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
Review comment:
Sure. Let's open a new Jira.
> I think we should make it dependent on Ratis Leader election timeout.
Do you mean for interval check?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583448845
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
##########
@@ -179,25 +193,79 @@ public long takeSnapshot() throws IOException {
long startTime = Time.monotonicNow();
TermIndex lastTermIndex = getLastAppliedTermIndex();
long lastAppliedIndex = lastTermIndex.getIndex();
- TransactionInfo lastAppliedTrxInfo =
- TransactionInfo.fromTermIndex(lastTermIndex);
- if (transactionBuffer.getLatestTrxInfo()
- .compareTo(lastAppliedTrxInfo) < 0) {
+ if (isInitialized.get()) {
+ TransactionInfo lastAppliedTrxInfo =
+ TransactionInfo.fromTermIndex(lastTermIndex);
+ if (transactionBuffer.getLatestTrxInfo().compareTo(lastAppliedTrxInfo)
+ < 0) {
+ transactionBuffer.updateLatestTrxInfo(
+ TransactionInfo.builder().setCurrentTerm(lastTermIndex.getTerm())
+ .setTransactionIndex(lastTermIndex.getIndex()).build());
+ transactionBuffer.setLatestSnapshot(
+ transactionBuffer.getLatestTrxInfo().toSnapshotInfo());
+ } else {
+ lastAppliedIndex =
+ transactionBuffer.getLatestTrxInfo().getTransactionIndex();
+ }
+
+ transactionBuffer.flush();
+ LOG.info("Current Snapshot Index {}, takeSnapshot took {} ms",
+ lastAppliedIndex, Time.monotonicNow() - startTime);
+ }
+ super.takeSnapshot();
Review comment:
yes, to ensure if there are all updates to sm directory by the base stateMachine impelementation (if any).
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant edited a comment on pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant edited a comment on pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#issuecomment-787475756
> Overall LGTM. One comment regarding usage of clientID and callID usage need to be set from Server API's that looks missing.
> And also getting localNodeID need to be fetched, as setNodeSpecificConfigKeys does not ozone.scm.node.id.
There is only one central place of submitting ratis requests and there is only client Id which will be same for all requests submitted to the server on each node. I don't think , we need to get the clientId from server here as there is only one client.
Same goes for "callId" as well as there is no retry logic as inside SCM for ratis requests.
However, this change is not related to the patch. We can discuss it further and make change accordingly, in a different jira.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583563075
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
##########
@@ -63,69 +61,120 @@
private static final Logger LOG =
LoggerFactory.getLogger(SCMRatisServerImpl.class);
- private final RaftServer.Division division;
+ private final RaftServer server;
+ private final SCMStateMachine stateMachine;
private final StorageContainerManager scm;
- private final InetSocketAddress address;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong callId = new AtomicLong();
// TODO: Refactor and remove ConfigurationSource and use only
// SCMHAConfiguration.
- SCMRatisServerImpl(final SCMHAConfiguration haConf,
- final ConfigurationSource conf, final StorageContainerManager scm,
- final DBTransactionBuffer buffer) throws IOException {
+ SCMRatisServerImpl(final ConfigurationSource conf,
+ final StorageContainerManager scm, final DBTransactionBuffer buffer)
+ throws IOException {
this.scm = scm;
- this.address = haConf.getRatisBindAddress();
- RaftServer server = newRaftServer(scm.getClusterId(), scm.getScmId(),
- scm.getSCMHANodeDetails(), conf)
- .setStateMachine(new SCMStateMachine(scm, this, buffer)).build();
+ this.stateMachine = new SCMStateMachine(scm, this, buffer);
+ final RaftGroupId groupId = buildRaftGroupId(scm.getClusterId());
+ LOG.info("starting Raft server for scm:{}", scm.getScmId());
+ // During SCM startup, the bootstrapped node will be started just with
+ // groupId information, so that it won't trigger any leader election
+ // as it doesn't have any peer info.
+
+ // The primary SCM node which is initialized using scm --init command,
+ // will initialize the raft server with the peer info and it will be
+ // persisted in the raft log post leader election. Now, when the primary
+ // scm boots up, it has peer info embedded in the raft log and will
+ // trigger leader election.
+ this.server =
+ newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine)
+ .setGroup(RaftGroup.valueOf(groupId)).build();
+ }
+
+ public static void initialize(String clusterId, String scmId,
+ SCMNodeDetails details, OzoneConfiguration conf) throws IOException {
+ final RaftGroup group = buildRaftGroup(details, scmId, clusterId);
+ RaftServer server = newRaftServer(scmId, conf)
+ .setGroup(group).build();
+ server.start();
+ waitforLeaderToBeReady(server, 60000, group);
+ server.close();
+ }
+
+ private static void waitforLeaderToBeReady(RaftServer server, long timeout,
+ RaftGroup group)
+ throws IOException {
+ boolean ready;
+ long st = Time.monotonicNow();
+ do {
+ ready = server.getDivision(group.getGroupId()).getInfo().isLeaderReady();
+ if (!ready) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } while (!ready && Time.monotonicNow() - st < timeout);
- this.division =
- server.getDivision(server.getGroups().iterator().next().getGroupId());
+ if (!ready) {
+ throw new IOException(String
+ .format("Ratis group %s is not ready in %d ms", group.getGroupId(),
+ timeout));
+ }
}
- public static RaftServer.Builder newRaftServer(final String clusterId,
- final String scmId, final SCMHANodeDetails haDetails,
- final ConfigurationSource conf) throws IOException {
- final String scmNodeId = haDetails.getLocalNodeDetails().getNodeId();
+ private static RaftServer.Builder newRaftServer(final String scmId,
+ final ConfigurationSource conf) {
final SCMHAConfiguration haConf = conf.getObject(SCMHAConfiguration.class);
- SCMHAGroupBuilder haGrpBuilder = scmNodeId != null ?
- new SCMHAGroupBuilder(haDetails, clusterId, scmId) :
- new SCMHAGroupBuilder(haConf, conf, clusterId);
final RaftProperties serverProperties =
RatisUtil.newRaftProperties(haConf, conf);
- return RaftServer.newBuilder().setServerId(haGrpBuilder.getPeerId())
- .setGroup(haGrpBuilder.getRaftGroup()).setProperties(serverProperties)
- .setStateMachine(new SCMStateMachine());
+ return RaftServer.newBuilder().setServerId(RaftPeerId.getRaftPeerId(scmId))
+ .setProperties(serverProperties)
+ .setStateMachine(new SCMStateMachine(false));
}
@Override
public void start() throws IOException {
- division.getRaftServer().start();
+ server.start();
+ }
+
+ public RaftServer.Division getDivision() {
+ try {
+ return server
+ .getDivision(server.getGroups().iterator().next().getGroupId());
+ } catch (Exception e) {
+ LOG.warn("Failed to get RaftServerDivision", e);
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ public SCMStateMachine getStateMachine() {
+ return stateMachine;
}
@Override
public void registerStateMachineHandler(final RequestType handlerType,
final Object handler) {
- ((SCMStateMachine) division.getStateMachine())
- .registerHandler(handlerType, handler);
+ stateMachine.registerHandler(handlerType, handler);
}
@Override
public SCMRatisResponse submitRequest(SCMRatisRequest request)
throws IOException, ExecutionException, InterruptedException {
final RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder()
.setClientId(clientId)
- .setServerId(division.getId())
- .setGroupId(division.getGroup().getGroupId())
+ .setServerId(getDivision().getId())
Review comment:
Division info can change depending upon server state. Its safe to use this via Ratis. Its a cached object away inside ratisServerImpl.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#issuecomment-787536854
@bharatviswa504 , i have filed https://issues.apache.org/jira/browse/HDDS-4877 and https://issues.apache.org/jira/browse/HDDS-4878 to address the issues discussed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] bshashikant commented on a change in pull request #1953: HDDS-4718. Bootstrap new SCM node
Posted by GitBox <gi...@apache.org>.
bshashikant commented on a change in pull request #1953:
URL: https://github.com/apache/ozone/pull/1953#discussion_r583477482
##########
File path: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
##########
@@ -1169,11 +1168,12 @@ public ReplicationManager getReplicationManager() {
}
/**
- * Check if the current scm is the leader.
- * @return - if the current scm is the leader.
+ * Check if the current scm is the leader and ready for accepting requests.
+ * @return - if the current scm is the leader and is ready.
*/
public boolean checkLeader() {
- return scmContext.isLeader();
+ return scmContext.isLeader() && getScmHAManager().getRatisServer()
Review comment:
Its for a case, where ratis server is not initialised yet. By default scmContext.isLeader() will be false.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org