You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bo...@apache.org on 2018/11/13 00:22:36 UTC
[16/50] [abbrv] hadoop git commit: HDDS-806. Update Ratis to latest
snapshot version in ozone. Contributed by Tsz Wo Nicholas Sze and Mukul Kumar
Singh.
HDDS-806. Update Ratis to latest snapshot version in ozone. Contributed by Tsz Wo Nicholas Sze and Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31614bcc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31614bcc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31614bcc
Branch: refs/heads/YARN-7402
Commit: 31614bcc7cda614c45769aa779a839b25c375db2
Parents: 8d99648
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Fri Nov 9 00:05:45 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Fri Nov 9 00:05:45 2018 +0530
----------------------------------------------------------------------
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 ++++
.../apache/hadoop/ozone/OzoneConfigKeys.java | 10 ++++
.../common/src/main/resources/ozone-default.xml | 15 ++++++
.../server/ratis/ContainerStateMachine.java | 48 +++++++++-----------
.../server/ratis/XceiverServerRatis.java | 11 +++++
hadoop-hdds/pom.xml | 2 +-
hadoop-ozone/pom.xml | 2 +-
7 files changed, 67 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31614bcc/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 38eec61..cedcc43 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -85,6 +85,14 @@ public final class ScmConfigKeys {
public static final TimeDuration
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
TimeDuration.valueOf(10, TimeUnit.SECONDS);
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES =
+ "dfs.container.ratis.statemachinedata.sync.retries";
+ public static final int
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT = -1;
+ public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE =
+ "dfs.container.ratis.log.queue.size";
+ public static final int DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE_DEFAULT = 128;
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
"dfs.ratis.client.request.timeout.duration";
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31614bcc/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 54b1cf8..9776817 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -261,6 +261,16 @@ public final class OzoneConfigKeys {
public static final TimeDuration
DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
ScmConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT;
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES;
+ public static final int
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT;
+ public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE;
+ public static final int DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE_DEFAULT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE_DEFAULT;
public static final String DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY =
ScmConfigKeys.DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY;
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31614bcc/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 5ff60eb..2ffc2ab 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -60,6 +60,21 @@
</description>
</property>
<property>
+ <name>dfs.container.ratis.statemachinedata.sync.retries</name>
+ <value>-1</value>
+ <tag>OZONE, DEBUG, CONTAINER, RATIS</tag>
+ <description>Number of times the WriteStateMachineData op will be tried
+ before failing, if this value is -1, then this retries indefinitely.
+ </description>
+ </property>
+ <property>
+ <name>dfs.container.ratis.log.queue.size</name>
+ <value>128</value>
+ <tag>OZONE, DEBUG, CONTAINER, RATIS</tag>
+ <description>Number of operation pending with Raft's Log Worker.
+ </description>
+ </property>
+ <property>
<name>dfs.container.ratis.datanode.storage.dir</name>
<value/>
<tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31614bcc/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 2a4a227..38b789e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -22,7 +22,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
-import org.apache.ratis.proto.RaftProtos.StateMachineEntryProto;
+import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.server.RaftServer;
@@ -55,7 +55,6 @@ import org.apache.ratis.statemachine.StateMachineStorage;
import org.apache.ratis.statemachine.TransactionContext;
import org.apache.ratis.statemachine.impl.BaseStateMachine;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
-import org.apache.ratis.statemachine.impl.TransactionContextImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -206,7 +205,6 @@ public class ContainerStateMachine extends BaseStateMachine {
final ContainerCommandRequestProto proto =
getRequestProto(request.getMessage().getContent());
Preconditions.checkArgument(request.getRaftGroupId().equals(gid));
- final StateMachineLogEntryProto log;
if (proto.getCmdType() == Type.WriteChunk) {
final WriteChunkRequestProto write = proto.getWriteChunk();
// create the state machine data proto
@@ -236,33 +234,29 @@ public class ContainerStateMachine extends BaseStateMachine {
.setWriteChunk(commitWriteChunkProto)
.build();
- log = createSMLogEntryProto(request,
- commitContainerCommandProto.toByteString(),
- dataContainerCommandProto.toByteString());
+ return TransactionContext.newBuilder()
+ .setClientRequest(request)
+ .setStateMachine(this)
+ .setServerRole(RaftPeerRole.LEADER)
+ .setStateMachineData(dataContainerCommandProto.toByteString())
+ .setLogData(commitContainerCommandProto.toByteString())
+ .build();
} else if (proto.getCmdType() == Type.CreateContainer) {
- log = createSMLogEntryProto(request,
- request.getMessage().getContent(), request.getMessage().getContent());
+ return TransactionContext.newBuilder()
+ .setClientRequest(request)
+ .setStateMachine(this)
+ .setServerRole(RaftPeerRole.LEADER)
+ .setStateMachineData(request.getMessage().getContent())
+ .setLogData(request.getMessage().getContent())
+ .build();
} else {
- log = createSMLogEntryProto(request, request.getMessage().getContent(),
- null);
+ return TransactionContext.newBuilder()
+ .setClientRequest(request)
+ .setStateMachine(this)
+ .setServerRole(RaftPeerRole.LEADER)
+ .setLogData(request.getMessage().getContent())
+ .build();
}
- return new TransactionContextImpl(this, request, log);
- }
-
- private StateMachineLogEntryProto createSMLogEntryProto(RaftClientRequest r,
- ByteString logData, ByteString smData) {
- StateMachineLogEntryProto.Builder builder =
- StateMachineLogEntryProto.newBuilder();
-
- builder.setCallId(r.getCallId())
- .setClientId(r.getClientId().toByteString())
- .setLogData(logData);
-
- if (smData != null) {
- builder.setStateMachineEntry(StateMachineEntryProto.newBuilder()
- .setStateMachineData(smData).build());
- }
- return builder.build();
}
private ByteString getStateMachineData(StateMachineLogEntryProto entryProto) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31614bcc/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index a679e5e..7bf4da9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -287,6 +287,17 @@ public final class XceiverServerRatis implements XceiverServerSpi {
setAutoTriggerEnabled(properties, true);
RaftServerConfigKeys.Snapshot.
setAutoTriggerThreshold(properties, snapshotThreshold);
+ int logQueueSize =
+ conf.getInt(OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE_DEFAULT);
+ RaftServerConfigKeys.Log.setQueueSize(properties, logQueueSize);
+
+ int numSyncRetries = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES,
+ OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT);
+ RaftServerConfigKeys.Log.StateMachineData.setSyncTimeoutRetry(properties,
+ numSyncRetries);
return properties;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31614bcc/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 090a537..7a1704c 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -45,7 +45,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<hdds.version>0.4.0-SNAPSHOT</hdds.version>
<!-- Apache Ratis version -->
- <ratis.version>0.3.0-1d2ebee-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-1d07b18-SNAPSHOT</ratis.version>
<bouncycastle.version>1.60</bouncycastle.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31614bcc/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 33af31b..671421e 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -33,7 +33,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<hadoop.version>3.2.1-SNAPSHOT</hadoop.version>
<hdds.version>0.4.0-SNAPSHOT</hdds.version>
<ozone.version>0.4.0-SNAPSHOT</ozone.version>
- <ratis.version>0.3.0-1d2ebee-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-1d07b18-SNAPSHOT</ratis.version>
<bouncycastle.version>1.60</bouncycastle.version>
<ozone.release>Badlands</ozone.release>
<declared.ozone.version>${ozone.version}</declared.ozone.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org