You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by na...@apache.org on 2020/10/25 17:23:51 UTC
[hadoop-ozone] 05/11: HDDS-4130. remove the 1st edition of
RatisServer of SCM HA which is copied from OM HA.
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit a70964ec84ac04ca3f0dbad9dd7b43f0f0807707
Author: Glen Geng <gl...@tencent.com>
AuthorDate: Sat Oct 24 21:04:32 2020 +0530
HDDS-4130. remove the 1st edition of RatisServer of SCM HA which is copied from OM HA.
---
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 14 --
.../hdds/scm/server/StorageContainerManager.java | 78 +----------
.../hdds/scm/server/ratis/SCMStateMachine.java | 6 +-
.../hdds/scm/server/ratis/TestSCMRatisServer.java | 147 ---------------------
.../hdds/scm/server/ratis/TestSCMStateMachine.java | 120 -----------------
5 files changed, 5 insertions(+), 360 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
index 0f71744..48946b4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
@@ -19,13 +19,9 @@
package org.apache.hadoop.hdds.scm.ha;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.ScmUtils;
-import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisServer;
-import java.io.File;
import java.util.Collection;
/**
@@ -42,16 +38,6 @@ public final class SCMHAUtils {
ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT);
}
- public static File createSCMRatisDir(ConfigurationSource conf)
- throws IllegalArgumentException {
- String scmRatisDir = SCMRatisServer.getSCMRatisDirectory(conf);
- if (scmRatisDir == null || scmRatisDir.isEmpty()) {
- throw new IllegalArgumentException(HddsConfigKeys.OZONE_METADATA_DIRS +
- " must be defined.");
- }
- return ScmUtils.createSCMDir(scmRatisDir);
- }
-
/**
* Get a collection of all scmNodeIds for the given scmServiceId.
*/
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index cbd713c..b17729b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -31,13 +31,11 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.protobuf.BlockingService;
-import java.io.File;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
-import java.util.Collections;
import java.util.Objects;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
@@ -54,10 +52,6 @@ import org.apache.hadoop.hdds.scm.PipelineChoosePolicy;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SCMHAManagerImpl;
-import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
-import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails;
-import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisServer;
-import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisSnapshotInfo;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -128,7 +122,6 @@ import org.apache.hadoop.util.JvmPauseMonitor;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT_DEFAULT;
import org.apache.ratis.grpc.GrpcTlsConfig;
-import org.apache.ratis.server.protocol.TermIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -202,11 +195,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private CertificateServer certificateServer;
private GrpcTlsConfig grpcTlsConfig;
- // SCM HA related
- private SCMRatisServer scmRatisServer;
- private SCMRatisSnapshotInfo scmRatisSnapshotInfo;
- private File scmRatisSnapshotDir;
-
private JvmPauseMonitor jvmPauseMonitor;
private final OzoneConfiguration configuration;
private SCMContainerMetrics scmContainerMetrics;
@@ -276,10 +264,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
loginAsSCMUser(conf);
}
- this.scmRatisSnapshotInfo = new SCMRatisSnapshotInfo(
- scmStorageConfig.getCurrentDir());
- this.scmRatisSnapshotDir = SCMHAUtils.createSCMRatisDir(conf);
-
// Creates the SCM DBs or opens them if it exists.
// A valid pointer to the store is required by all the other services below.
initalizeMetadataStore(conf, configurator);
@@ -400,13 +384,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
public static StorageContainerManager createSCM(
OzoneConfiguration conf, SCMConfigurator configurator)
throws IOException, AuthenticationException {
- StorageContainerManager scm = new StorageContainerManager(
- conf, configurator);
- if (SCMHAUtils.isSCMHAEnabled(conf) && scm.getScmRatisServer() == null) {
- SCMRatisServer scmRatisServer = initializeRatisServer(conf, scm);
- scm.setScmRatisServer(scmRatisServer);
- }
- return scm;
+ return new StorageContainerManager(conf, configurator);
}
/**
@@ -840,10 +818,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
getClientRpcAddress()));
}
- if (scmRatisServer != null) {
- scmRatisServer.start();
- }
-
scmHAManager.start();
ms = HddsServerUtil
@@ -1180,56 +1154,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return this.clusterMap;
}
- private static SCMRatisServer initializeRatisServer(
- OzoneConfiguration conf, StorageContainerManager scm) throws IOException {
- SCMNodeDetails scmNodeDetails = SCMNodeDetails
- .initStandAlone(conf);
- //TODO enable Ratis group
- SCMRatisServer scmRatisServer = SCMRatisServer.newSCMRatisServer(
- conf.getObject(SCMRatisServer.SCMRatisServerConfiguration.class),
- scm, scmNodeDetails, Collections.EMPTY_LIST,
- SCMRatisServer.getSCMRatisDirectory(conf));
- if (scmRatisServer != null) {
- LOG.info("SCM Ratis server initialized at port {}",
- scmRatisServer.getServerPort());
- } // TODO error handling for scmRatisServer creation failure
- return scmRatisServer;
- }
-
- @VisibleForTesting
- public SCMRatisServer getScmRatisServer() {
- return scmRatisServer;
- }
-
- public void setScmRatisServer(SCMRatisServer scmRatisServer) {
- this.scmRatisServer = scmRatisServer;
- }
-
- @VisibleForTesting
- public SCMRatisSnapshotInfo getSnapshotInfo() {
- return scmRatisSnapshotInfo;
- }
-
- @VisibleForTesting
- public long getRatisSnapshotIndex() {
- return scmRatisSnapshotInfo.getIndex();
- }
-
- /**
- * Save ratis snapshot to SCM meta store and local disk.
- */
- public TermIndex saveRatisSnapshot() throws IOException {
- TermIndex snapshotIndex = scmRatisServer.getLastAppliedTermIndex();
- if (scmMetadataStore != null) {
- // Flush the SCM state to disk
- scmMetadataStore.getStore().flushDB();
- }
-
- scmRatisSnapshotInfo.saveRatisSnapshotToDisk(snapshotIndex);
-
- return snapshotIndex;
- }
-
/**
* Get the safe mode status of all rules.
*
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java
index 144380a..9a725a6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java
@@ -73,7 +73,8 @@ public class SCMStateMachine extends BaseStateMachine {
this.scmRatisServer = ratisServer;
this.scm = ratisServer.getSCM();
- this.snapshotInfo = scm.getSnapshotInfo();
+ // TODO: remove the whole file later
+ this.snapshotInfo = null;
updateLastAppliedIndexWithSnaphsotIndex();
ThreadFactory build = new ThreadFactoryBuilder().setDaemon(true)
@@ -158,7 +159,8 @@ public class SCMStateMachine extends BaseStateMachine {
public long takeSnapshot() throws IOException {
LOG.info("Saving Ratis snapshot on the SCM.");
if (scm != null) {
- return scm.saveRatisSnapshot().getIndex();
+ // TODO: remove the whole file later
+ return 0;
}
return 0;
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java
deleted file mode 100644
index 8a233ae..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.ratis;
-
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails;
-import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.ratis.protocol.RaftGroupId;
-import org.apache.ratis.util.LifeCycle;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Collections;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
-
-/**
- * Test class for SCM Ratis Server.
- */
-public class TestSCMRatisServer {
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- private OzoneConfiguration conf;
- private SCMRatisServer scmRatisServer;
- private StorageContainerManager scm;
- private String scmId;
- private static final long LEADER_ELECTION_TIMEOUT = 500L;
-
- @Before
- public void init() throws Exception {
- conf = new OzoneConfiguration();
- scmId = UUID.randomUUID().toString();
- conf.setTimeDuration(
- ScmConfigKeys.OZONE_SCM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
- LEADER_ELECTION_TIMEOUT, TimeUnit.MILLISECONDS);
- conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
- conf.set(ScmConfigKeys.OZONE_SCM_INTERNAL_SERVICE_ID, "scm-ha-test");
-
- // Standalone SCM Ratis server
- initSCM();
- scm = TestUtils.getScm(conf);
- scm.start();
- scmRatisServer = scm.getScmRatisServer();
- }
-
- @After
- public void shutdown() {
- if (scmRatisServer != null) {
- scmRatisServer.stop();
- }
- if (scm != null) {
- scm.stop();
- }
- }
-
- @Test
- public void testStartSCMRatisServer() {
- Assert.assertEquals("Ratis Server should be in running state",
- LifeCycle.State.RUNNING, scmRatisServer.getServerState());
- }
-
- @Test
- public void verifyRaftGroupIdGenerationWithCustomOmServiceId() throws
- Exception {
- String customScmServiceId = "scmIdCustom123";
- OzoneConfiguration newConf = new OzoneConfiguration();
- String newOmId = UUID.randomUUID().toString();
- String path = GenericTestUtils.getTempPath(newOmId);
- Path metaDirPath = Paths.get(path, "scm-meta");
- newConf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
- newConf.setTimeDuration(
- ScmConfigKeys.OZONE_SCM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
- LEADER_ELECTION_TIMEOUT, TimeUnit.MILLISECONDS);
- int ratisPort = 9873;
- InetSocketAddress rpcAddress = new InetSocketAddress(
- InetAddress.getLocalHost(), 0);
- SCMNodeDetails nodeDetails = new SCMNodeDetails.Builder()
- .setRpcAddress(rpcAddress)
- .setRatisPort(ratisPort)
- .setSCMNodeId(newOmId)
- .setSCMServiceId(customScmServiceId)
- .build();
- // Starts a single node Ratis server
- scmRatisServer.stop();
- SCMRatisServer newScmRatisServer = SCMRatisServer
- .newSCMRatisServer(newConf.getObject(SCMRatisServer
- .SCMRatisServerConfiguration.class), scm, nodeDetails,
- Collections.emptyList(),
- SCMRatisServer.getSCMRatisDirectory(newConf));
- newScmRatisServer.start();
-
- UUID uuid = UUID.nameUUIDFromBytes(customScmServiceId.getBytes());
- RaftGroupId raftGroupId = newScmRatisServer.getRaftGroup().getGroupId();
- Assert.assertEquals(uuid, raftGroupId.getUuid());
- Assert.assertEquals(raftGroupId.toByteString().size(), 16);
- newScmRatisServer.stop();
- }
-
- private void initSCM() throws IOException {
- String clusterId = UUID.randomUUID().toString();
- scmId = UUID.randomUUID().toString();
-
- final String path = folder.newFolder().toString();
- Path scmPath = Paths.get(path, "scm-meta");
- Files.createDirectories(scmPath);
- conf.set(OZONE_METADATA_DIRS, scmPath.toString());
- SCMStorageConfig scmStore = new SCMStorageConfig(conf);
- scmStore.setClusterId(clusterId);
- scmStore.setScmId(scmId);
- // writes the version file properties
- scmStore.initialize();
- }
-}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java
deleted file mode 100644
index 0eddbde..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.server.ratis;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.UUID;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
-
-/**
- * Test class for SCMStateMachine.
- */
-public class TestSCMStateMachine {
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- private SCMStateMachine scmStateMachine;
- private StorageContainerManager scm;
- private SCMRatisServer scmRatisServer;
- private OzoneConfiguration conf;
- private String scmId;
- @Before
- public void init() throws Exception {
- conf = new OzoneConfiguration();
- conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
- conf.set(ScmConfigKeys.OZONE_SCM_INTERNAL_SERVICE_ID, "scm-ha-test");
- scmId = UUID.randomUUID().toString();
-
- initSCM();
- scm = TestUtils.getScm(conf);
- scm.start();
- scmRatisServer = scm.getScmRatisServer();
- scmStateMachine = scm.getScmRatisServer().getScmStateMachine();
- }
-
- @Test
- public void testSCMUpdatedAppliedIndex(){
- // State machine should start with 0 term and 0 index.
- scmStateMachine.notifyIndexUpdate(0, 0);
- Assert.assertEquals(0,
- scmStateMachine.getLastAppliedTermIndex().getTerm());
- Assert.assertEquals(0,
- scmStateMachine.getLastAppliedTermIndex().getIndex());
-
- // If only the transactionMap is updated, index should stay 0.
- scmStateMachine.addApplyTransactionTermIndex(0, 1);
- Assert.assertEquals(0L,
- scmStateMachine.getLastAppliedTermIndex().getTerm());
- Assert.assertEquals(0L,
- scmStateMachine.getLastAppliedTermIndex().getIndex());
-
- // After the index update is notified, the index should increase.
- scmStateMachine.notifyIndexUpdate(0, 1);
- Assert.assertEquals(0L,
- scmStateMachine.getLastAppliedTermIndex().getTerm());
- Assert.assertEquals(1L,
- scmStateMachine.getLastAppliedTermIndex().getIndex());
-
- // Only do a notifyIndexUpdate can also increase the index.
- scmStateMachine.notifyIndexUpdate(0, 2);
- Assert.assertEquals(0L,
- scmStateMachine.getLastAppliedTermIndex().getTerm());
- Assert.assertEquals(2L,
- scmStateMachine.getLastAppliedTermIndex().getIndex());
-
- // If a larger index is notified, the index should not be updated.
- scmStateMachine.notifyIndexUpdate(0, 5);
- Assert.assertEquals(0L,
- scmStateMachine.getLastAppliedTermIndex().getTerm());
- Assert.assertEquals(2L,
- scmStateMachine.getLastAppliedTermIndex().getIndex());
- }
-
- private void initSCM() throws IOException {
- String clusterId = UUID.randomUUID().toString();
- final String path = folder.newFolder().toString();
- Path scmPath = Paths.get(path, "scm-meta");
- Files.createDirectories(scmPath);
- conf.set(OZONE_METADATA_DIRS, scmPath.toString());
- SCMStorageConfig scmStore = new SCMStorageConfig(conf);
- scmStore.setClusterId(clusterId);
- scmStore.setScmId(scmId);
- // writes the version file properties
- scmStore.initialize();
- }
-
- @After
- public void cleanup() {
- scm.stop();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org