You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2018/11/07 02:09:12 UTC
[14/37] hadoop git commit: HDDS-794. Add configs to set
StateMachineData write timeout in ContainerStateMachine. Contributed by
Shashikant Banerjee.
HDDS-794. Add configs to set StateMachineData write timeout in ContainerStateMachine. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ddefdd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ddefdd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ddefdd5
Branch: refs/heads/HDFS-13891
Commit: 5ddefdd50751ed316f2eb9046f294bbdcdfb2428
Parents: 942693b
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Nov 5 10:10:10 2018 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Nov 5 10:41:28 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 ++++++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 9 +++++++++
.../common/src/main/resources/ozone-default.xml | 7 +++++++
.../server/ratis/ContainerStateMachine.java | 18 ++++++++++++++++--
.../server/ratis/XceiverServerRatis.java | 14 ++++++++++++++
.../container/keyvalue/helpers/ChunkUtils.java | 2 ++
.../container/keyvalue/impl/ChunkManagerImpl.java | 3 ++-
7 files changed, 56 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 56692af..38eec61 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -79,6 +79,12 @@ public final class ScmConfigKeys {
"dfs.container.ratis.segment.preallocated.size";
public static final int
DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
+ "dfs.container.ratis.statemachinedata.sync.timeout";
+ public static final TimeDuration
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
+ TimeDuration.valueOf(10, TimeUnit.SECONDS);
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
"dfs.ratis.client.request.timeout.duration";
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 3b4f017..54b1cf8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -229,6 +229,15 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
+
+ // config settings to enable stateMachineData write timeout
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT;
+ public static final TimeDuration
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
+
public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
= ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index eb68662..5ff60eb 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -53,6 +53,13 @@
</description>
</property>
<property>
+ <name>dfs.container.ratis.statemachinedata.sync.timeout</name>
+ <value>10s</value>
+ <tag>OZONE, DEBUG, CONTAINER, RATIS</tag>
+ <description>Timeout for StateMachine data writes by Ratis.
+ </description>
+ </property>
+ <property>
<name>dfs.container.ratis.datanode.storage.dir</name>
<value/>
<tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index d5762bc..2a4a227 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -307,9 +307,18 @@ public class ContainerStateMachine extends BaseStateMachine {
() -> runCommand(requestProto), chunkExecutor);
}
writeChunkFutureMap.put(entryIndex, writeChunkFuture);
+ LOG.debug("writeChunk writeStateMachineData : blockId " + write.getBlockID()
+ + " logIndex " + entryIndex + " chunkName " + write.getChunkData()
+ .getChunkName());
// Remove the future once it finishes execution from the
// writeChunkFutureMap.
- writeChunkFuture.thenApply(r -> writeChunkFutureMap.remove(entryIndex));
+ writeChunkFuture.thenApply(r -> {
+ writeChunkFutureMap.remove(entryIndex);
+ LOG.debug("writeChunk writeStateMachineData completed: blockId " + write
+ .getBlockID() + " logIndex " + entryIndex + " chunkName " + write
+ .getChunkData().getChunkName());
+ return r;
+ });
return writeChunkFuture;
}
@@ -531,7 +540,12 @@ public class ContainerStateMachine extends BaseStateMachine {
if (cmdType == Type.CreateContainer) {
long containerID = requestProto.getContainerID();
future.thenApply(
- r -> createContainerFutureMap.remove(containerID).complete(null));
+ r -> {
+ createContainerFutureMap.remove(containerID).complete(null);
+ LOG.info("create Container Transaction completed for container " +
+ containerID + " log index " + index);
+ return r;
+ });
}
future.thenAccept(m -> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 599f821..a679e5e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -186,6 +186,20 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftClientConfigKeys.Rpc
.setRequestTimeout(properties, clientRequestTimeout);
+ // set the configs enable and set the stateMachineData sync timeout
+ RaftServerConfigKeys.Log.StateMachineData.setSync(properties, true);
+ timeUnit = OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT.getUnit();
+ duration = conf.getTimeDuration(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT,
+ OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT
+ .getDuration(), timeUnit);
+ final TimeDuration dataSyncTimeout =
+ TimeDuration.valueOf(duration, timeUnit);
+ RaftServerConfigKeys.Log.StateMachineData
+ .setSyncTimeout(properties, dataSyncTimeout);
+
// Set the server Request timeout
timeUnit = OzoneConfigKeys.DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_DEFAULT
.getUnit();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 718f5de..8f9d589 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -138,6 +138,8 @@ public final class ChunkUtils {
}
}
}
+ log.debug("Write Chunk completed for chunkFile: {}, size {}", chunkFile,
+ data.length);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index c630e19..cdd19df 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -82,7 +82,8 @@ public class ChunkManagerImpl implements ChunkManager {
chunkFile, info);
File tmpChunkFile = getTmpChunkFile(chunkFile, info);
- LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file",
+ LOG.debug(
+ "writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file:{}",
info.getChunkName(), stage, chunkFile, tmpChunkFile);
switch (stage) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org