You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by li...@apache.org on 2020/06/02 03:11:12 UTC
[hadoop-ozone] 03/07: Resolve conflicts with merge from master.
This is an automated email from the ASF dual-hosted git repository.
licheng pushed a commit to branch HDDS-2823
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit a8ac3b27fcc8f1926107bbf8b6481eb7b16da5f1
Author: Li Cheng <ti...@tencent.com>
AuthorDate: Thu Apr 30 16:26:03 2020 +0800
Resolve conflicts with merge from master.
---
.../hdds/scm/container/ContainerManager.java | 5 ---
.../hdds/scm/container/SCMContainerManager.java | 7 ----
.../hdds/scm/server/StorageContainerManager.java | 4 --
.../hdds/scm/server/ratis/SCMRatisServer.java | 44 +++++++++++-----------
.../hdds/scm/server/ratis/SCMStateMachine.java | 8 ++--
5 files changed, 26 insertions(+), 42 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
index f17a2f4..43c1ced 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
@@ -189,9 +189,4 @@ public interface ContainerManager extends Closeable {
* @param success
*/
void notifyContainerReportProcessing(boolean isFullReport, boolean success);
-
- /**
- * Flush metadata of container manager if they are required to be persisted.
- */
- void flushDB() throws IOException;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
index ee8c689..50b2829 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
@@ -365,13 +365,6 @@ public class SCMContainerManager implements ContainerManager {
}
}
- @Override
- public void flushDB() throws IOException {
- if (containerStore != null) {
- containerStore.flushDB(true);
- }
- }
-
/**
* Update deleteTransactionId according to deleteTransactionMap.
*
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index b8527bc..1a9b70c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -1176,10 +1176,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmMetadataStore.getStore().flush();
}
- if (containerManager != null) {
- containerManager.flushDB();
- }
-
scmRatisSnapshotInfo.saveRatisSnapshotToDisk(snapshotIndex);
return snapshotIndex;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMRatisServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMRatisServer.java
index 77dee6a..89a9d55 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMRatisServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMRatisServer.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hdds.scm.server.ratis;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -30,7 +30,8 @@ import org.apache.ratis.RaftConfigKeys;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
import org.apache.ratis.netty.NettyConfigKeys;
-import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
+import org.apache.ratis.proto.RaftProtos.RoleInfoProto;
import org.apache.ratis.protocol.ClientId;
import org.apache.ratis.protocol.GroupInfoReply;
import org.apache.ratis.protocol.GroupInfoRequest;
@@ -87,7 +88,7 @@ public final class SCMRatisServer {
private long roleCheckInitialDelayMs = 1000; // 1 second default
private long roleCheckIntervalMs;
private ReentrantReadWriteLock roleCheckLock = new ReentrantReadWriteLock();
- private Optional<RaftProtos.RaftPeerRole> cachedPeerRole = Optional.empty();
+ private Optional<RaftPeerRole> cachedPeerRole = Optional.empty();
private Optional<RaftPeerId> cachedLeaderPeerId = Optional.empty();
private static final AtomicLong CALL_ID_COUNTER = new AtomicLong();
@@ -99,7 +100,7 @@ public final class SCMRatisServer {
* Creates a SCM Ratis Server.
* @throws IOException
*/
- private SCMRatisServer(Configuration conf,
+ private SCMRatisServer(ConfigurationSource conf,
StorageContainerManager scm,
String raftGroupIdStr, RaftPeerId localRaftPeerId,
InetSocketAddress addr, List<RaftPeer> raftPeers)
@@ -137,7 +138,7 @@ public final class SCMRatisServer {
public void run() {
// Run this check only on the leader OM
if (cachedPeerRole.isPresent() &&
- cachedPeerRole.get() == RaftProtos.RaftPeerRole.LEADER) {
+ cachedPeerRole.get() == RaftPeerRole.LEADER) {
updateServerRole();
}
}
@@ -148,7 +149,7 @@ public final class SCMRatisServer {
* Create a SCM Ratis Server instance.
*/
public static SCMRatisServer newSCMRatisServer(
- Configuration conf, StorageContainerManager scm,
+ ConfigurationSource conf, StorageContainerManager scm,
SCMNodeDetails scmNodeDetails, List<SCMNodeDetails> peers)
throws IOException {
String scmServiceId = scmNodeDetails.getSCMServiceId();
@@ -186,7 +187,7 @@ public final class SCMRatisServer {
return new SCMStateMachine(this);
}
- private RaftProperties newRaftProperties(Configuration conf) {
+ private RaftProperties newRaftProperties(ConfigurationSource conf) {
final RaftProperties properties = new RaftProperties();
// Set RPC type
final String rpcType = conf.get(
@@ -202,20 +203,20 @@ public final class SCMRatisServer {
}
// Set Ratis storage directory
String storageDir = SCMRatisServer.getSCMRatisDirectory(conf);
- RaftServerConfigKeys.setStorageDirs(properties,
+ RaftServerConfigKeys.setStorageDir(properties,
Collections.singletonList(new File(storageDir)));
// Set RAFT segment size
final int raftSegmentSize = (int) conf.getStorageSize(
ScmConfigKeys.OZONE_SCM_RATIS_SEGMENT_SIZE_KEY,
ScmConfigKeys.OZONE_SCM_RATIS_SEGMENT_SIZE_DEFAULT,
- StorageUnit.BYTES);
+ org.apache.hadoop.hdds.conf.StorageUnit.BYTES);
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
SizeInBytes.valueOf(raftSegmentSize));
// Set RAFT segment pre-allocated size
final int raftSegmentPreallocatedSize = (int) conf.getStorageSize(
ScmConfigKeys.OZONE_SCM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
ScmConfigKeys.OZONE_SCM_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
- StorageUnit.BYTES);
+ org.apache.hadoop.hdds.conf.StorageUnit.BYTES);
int logAppenderQueueNumElements = conf.getInt(
ScmConfigKeys.OZONE_SCM_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS,
ScmConfigKeys.OZONE_SCM_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT);
@@ -280,7 +281,7 @@ public final class SCMRatisServer {
RaftServerConfigKeys.Rpc.setTimeoutMax(properties,
serverMaxTimeout);
// Set the number of maximum cached segments
- RaftServerConfigKeys.Log.setMaxCachedSegmentNum(properties, 2);
+ RaftServerConfigKeys.Log.setSegmentCacheNumMax(properties, 2);
// TODO: set max write buffer size
// Set the ratis leader election timeout
TimeUnit leaderElectionMinTimeoutUnit =
@@ -355,7 +356,7 @@ public final class SCMRatisServer {
this.roleCheckLock.readLock().lock();
try {
if (cachedPeerRole.isPresent() &&
- cachedPeerRole.get() == RaftProtos.RaftPeerRole.LEADER) {
+ cachedPeerRole.get() ==RaftPeerRole.LEADER) {
return true;
}
return false;
@@ -393,7 +394,7 @@ public final class SCMRatisServer {
/**
* Get the local directory where ratis logs will be stored.
*/
- public static String getSCMRatisDirectory(Configuration conf) {
+ public static String getSCMRatisDirectory(ConfigurationSource conf) {
String storageDir = conf.get(ScmConfigKeys.OZONE_SCM_RATIS_STORAGE_DIR);
if (Strings.isNullOrEmpty(storageDir)) {
@@ -427,17 +428,17 @@ public final class SCMRatisServer {
public void updateServerRole() {
try {
GroupInfoReply groupInfo = getGroupInfo();
- RaftProtos.RoleInfoProto roleInfoProto = groupInfo.getRoleInfoProto();
- RaftProtos.RaftPeerRole thisNodeRole = roleInfoProto.getRole();
+ RoleInfoProto roleInfoProto = groupInfo.getRoleInfoProto();
+ RaftPeerRole thisNodeRole = roleInfoProto.getRole();
- if (thisNodeRole.equals(RaftProtos.RaftPeerRole.LEADER)) {
+ if (thisNodeRole.equals(RaftPeerRole.LEADER)) {
setServerRole(thisNodeRole, raftPeerId);
- } else if (thisNodeRole.equals(RaftProtos.RaftPeerRole.FOLLOWER)) {
+ } else if (thisNodeRole.equals(RaftPeerRole.FOLLOWER)) {
ByteString leaderNodeId = roleInfoProto.getFollowerInfo()
.getLeaderInfo().getId().getId();
// There may be a chance, here we get leaderNodeId as null. For
- // example, in 3 node OM Ratis, if 2 SCM nodes are down, there will
+ // example, in 3 node OM Ratis, if 2 OM nodes are down, there will
// be no leader.
RaftPeerId leaderPeerId = null;
if (leaderNodeId != null && !leaderNodeId.isEmpty()) {
@@ -452,8 +453,7 @@ public final class SCMRatisServer {
}
} catch (IOException e) {
LOG.error("Failed to retrieve RaftPeerRole. Setting cached role to " +
- "{} and resetting leader info.",
- RaftProtos.RaftPeerRole.UNRECOGNIZED, e);
+ "{} and resetting leader info.", RaftPeerRole.UNRECOGNIZED, e);
setServerRole(null, null);
}
}
@@ -469,7 +469,7 @@ public final class SCMRatisServer {
return groupInfo;
}
- private void setServerRole(RaftProtos.RaftPeerRole currentRole,
+ private void setServerRole(RaftPeerRole currentRole,
RaftPeerId leaderPeerId) {
this.roleCheckLock.writeLock().lock();
try {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java
index b60570b..144380a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ratis/SCMStateMachine.java
@@ -88,7 +88,7 @@ public class SCMStateMachine extends BaseStateMachine {
@Override
public void initialize(RaftServer server, RaftGroupId id,
RaftStorage raftStorage) throws IOException {
- lifeCycle.startAndTransition(() -> {
+ getLifeCycle().startAndTransition(() -> {
super.initialize(server, id, raftStorage);
this.raftGroupId = id;
storage.init(raftStorage);
@@ -134,8 +134,8 @@ public class SCMStateMachine extends BaseStateMachine {
*/
@Override
public void pause() {
- lifeCycle.transition(LifeCycle.State.PAUSING);
- lifeCycle.transition(LifeCycle.State.PAUSED);
+ getLifeCycle().transition(LifeCycle.State.PAUSING);
+ getLifeCycle().transition(LifeCycle.State.PAUSED);
}
/**
@@ -144,7 +144,7 @@ public class SCMStateMachine extends BaseStateMachine {
*/
public void unpause(long newLastAppliedSnaphsotIndex,
long newLastAppliedSnapShotTermIndex) {
- lifeCycle.startAndTransition(() -> {
+ getLifeCycle().startAndTransition(() -> {
this.setLastAppliedTermIndex(TermIndex.newTermIndex(
newLastAppliedSnapShotTermIndex, newLastAppliedSnaphsotIndex));
});
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org