You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/11/05 20:42:57 UTC
[01/50] [abbrv] hadoop git commit: HDFS-14027. DFSStripedOutputStream
should implement both hsync methods.
Repository: hadoop
Updated Branches:
refs/heads/HDDS-4 2115256ab -> 7119be30b
HDFS-14027. DFSStripedOutputStream should implement both hsync methods.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db7e6368
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db7e6368
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db7e6368
Branch: refs/heads/HDDS-4
Commit: db7e636824a36b90ba1c8e9b2fba1162771700fe
Parents: 496f0ff
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Oct 29 19:05:52 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Oct 29 19:06:15 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/DFSStripedOutputStream.java | 12 +++++++
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 36 +++++++++++++-------
2 files changed, 35 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db7e6368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index ed875bb..df9770e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -956,11 +957,22 @@ public class DFSStripedOutputStream extends DFSOutputStream
@Override
public void hflush() {
// not supported yet
+ LOG.debug("DFSStripedOutputStream does not support hflush. "
+ + "Caller should check StreamCapabilities before calling.");
}
@Override
public void hsync() {
// not supported yet
+ LOG.debug("DFSStripedOutputStream does not support hsync. "
+ + "Caller should check StreamCapabilities before calling.");
+ }
+
+ @Override
+ public void hsync(EnumSet<SyncFlag> syncFlags) {
+ // not supported yet
+ LOG.debug("DFSStripedOutputStream does not support hsync {}. "
+ + "Caller should check StreamCapabilities before calling.", syncFlags);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db7e6368/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 865a736..092aa0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -18,12 +18,14 @@
package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.EnumSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,6 +33,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StreamCapabilities.StreamCapability;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.io.IOUtils;
@@ -196,19 +199,26 @@ public class TestDFSStripedOutputStream {
public void testStreamFlush() throws Exception {
final byte[] bytes = StripedFileTestUtil.generateBytes(blockSize *
dataBlocks * 3 + cellSize * dataBlocks + cellSize + 123);
- FSDataOutputStream os = fs.create(new Path("/ec-file-1"));
- assertFalse("DFSStripedOutputStream should not have hflush() " +
- "capability yet!", os.hasCapability(
- StreamCapability.HFLUSH.getValue()));
- assertFalse("DFSStripedOutputStream should not have hsync() " +
- "capability yet!", os.hasCapability(
- StreamCapability.HSYNC.getValue()));
- InputStream is = new ByteArrayInputStream(bytes);
- IOUtils.copyBytes(is, os, bytes.length);
- os.hflush();
- IOUtils.copyBytes(is, os, bytes.length);
- os.hsync();
- os.close();
+ try (FSDataOutputStream os = fs.create(new Path("/ec-file-1"))) {
+ assertFalse(
+ "DFSStripedOutputStream should not have hflush() capability yet!",
+ os.hasCapability(StreamCapability.HFLUSH.getValue()));
+ assertFalse(
+ "DFSStripedOutputStream should not have hsync() capability yet!",
+ os.hasCapability(StreamCapability.HSYNC.getValue()));
+ try (InputStream is = new ByteArrayInputStream(bytes)) {
+ IOUtils.copyBytes(is, os, bytes.length);
+ os.hflush();
+ IOUtils.copyBytes(is, os, bytes.length);
+ os.hsync();
+ IOUtils.copyBytes(is, os, bytes.length);
+ }
+ assertTrue("stream is not a DFSStripedOutputStream",
+ os.getWrappedStream() instanceof DFSStripedOutputStream);
+ final DFSStripedOutputStream dfssos =
+ (DFSStripedOutputStream) os.getWrappedStream();
+ dfssos.hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
+ }
}
private void testOneFile(String src, int writeBytes) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: YARN-6729. Clarify documentation
on how to enable cgroup support. Contributed by Zhankun Tang
Posted by ae...@apache.org.
YARN-6729. Clarify documentation on how to enable cgroup support. Contributed by Zhankun Tang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/277a3d8d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/277a3d8d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/277a3d8d
Branch: refs/heads/HDDS-4
Commit: 277a3d8d9fe1127c75452d083ff7859c603e686d
Parents: d36012b
Author: Shane Kumpf <sk...@apache.org>
Authored: Tue Oct 30 11:36:55 2018 -0600
Committer: Shane Kumpf <sk...@apache.org>
Committed: Tue Oct 30 11:36:55 2018 -0600
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/277a3d8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
index 4a83dce..7a48f6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCgroups.md
@@ -29,13 +29,13 @@ The following settings are related to setting up CGroups. These need to be set i
|Configuration Name | Description |
|:---- |:---- |
| `yarn.nodemanager.container-executor.class` | This should be set to "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor". CGroups is a Linux kernel feature and is exposed via the LinuxContainerExecutor. |
-| `yarn.nodemanager.linux-container-executor.resources-handler.class` | This should be set to "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler". Using the LinuxContainerExecutor doesn't force you to use CGroups. If you wish to use CGroups, the resource-handler-class must be set to CGroupsLCEResourceHandler. |
+| `yarn.nodemanager.linux-container-executor.resources-handler.class` | This should be set to "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler". Using the LinuxContainerExecutor doesn't force you to use CGroups. If you wish to use CGroups, the resource-handler-class must be set to CGroupsLCEResourceHandler. DefaultLCEResourcesHandler won't work. |
| `yarn.nodemanager.linux-container-executor.cgroups.hierarchy` | The cgroups hierarchy under which to place YARN proccesses(cannot contain commas). If yarn.nodemanager.linux-container-executor.cgroups.mount is false (that is, if cgroups have been pre-configured) and the YARN user has write access to the parent directory, then the directory will be created. If the directory already exists, the administrator has to give YARN write permissions to it recursively. |
| `yarn.nodemanager.linux-container-executor.cgroups.mount` | Whether the LCE should attempt to mount cgroups if not found - can be true or false. |
| `yarn.nodemanager.linux-container-executor.cgroups.mount-path` | Optional. Where CGroups are located. LCE will try to mount them here, if `yarn.nodemanager.linux-container-executor.cgroups.mount` is true. LCE will try to use CGroups from this location, if `yarn.nodemanager.linux-container-executor.cgroups.mount` is false. If specified, this path and its subdirectories (CGroup hierarchies) must exist and they should be readable and writable by YARN before the NodeManager is launched. See CGroups mount options below for details. |
| `yarn.nodemanager.linux-container-executor.group` | The Unix group of the NodeManager. It should match the setting in "container-executor.cfg". This configuration is required for validating the secure access of the container-executor binary. |
-The following settings are related to limiting resource usage of YARN containers:
+Once CGroups enabled, the following settings related to limiting resource usage of YARN containers can works:
|Configuration Name | Description |
|:---- |:---- |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HDDS-749. Restructure BlockId
class in Ozone. Contributed by Shashikant Banerjee.
Posted by ae...@apache.org.
HDDS-749. Restructure BlockId class in Ozone. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7757331d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7757331d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7757331d
Branch: refs/heads/HDDS-4
Commit: 7757331dbc043694891a5242ac161adece9e8d6a
Parents: 486b9a4
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Tue Oct 30 14:15:27 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Tue Oct 30 14:15:27 2018 +0530
----------------------------------------------------------------------
.../hdds/scm/storage/ChunkOutputStream.java | 17 ++--
.../org/apache/hadoop/hdds/client/BlockID.java | 85 ++++++++++++++------
.../hadoop/hdds/client/ContainerBlockID.java | 79 ++++++++++++++++++
.../common/helpers/AllocatedBlock.java | 21 ++---
...kLocationProtocolClientSideTranslatorPB.java | 5 +-
.../scm/storage/ContainerProtocolCalls.java | 7 +-
.../apache/hadoop/ozone/common/BlockGroup.java | 3 +-
.../container/common/helpers/BlockData.java | 8 +-
...kLocationProtocolServerSideTranslatorPB.java | 2 +-
.../main/proto/DatanodeContainerProtocol.proto | 4 +-
.../main/proto/ScmBlockLocationProtocol.proto | 2 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 7 +-
.../container/keyvalue/KeyValueHandler.java | 5 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 -
.../keyvalue/impl/BlockManagerImpl.java | 6 +-
.../keyvalue/interfaces/BlockManager.java | 3 +-
.../keyvalue/TestBlockManagerImpl.java | 6 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 3 +-
.../ozone/client/io/ChunkGroupInputStream.java | 3 +-
.../ozone/client/io/ChunkGroupOutputStream.java | 21 ++---
.../ozone/om/helpers/OmKeyLocationInfo.java | 19 +----
.../src/main/proto/OzoneManagerProtocol.proto | 1 -
.../container/TestContainerReplication.java | 2 +-
.../common/impl/TestCloseContainerHandler.java | 2 +-
.../common/impl/TestContainerPersistence.java | 14 ++--
.../TestGetCommittedBlockLengthAndPutKey.java | 16 ++--
.../hadoop/ozone/web/client/TestKeys.java | 2 +-
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 7 +-
.../ozone/om/ScmBlockLocationTestIngClient.java | 3 +-
29 files changed, 227 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
index 4547163..4e881c4 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.scm.storage;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.commons.codec.digest.DigestUtils;
@@ -57,7 +58,7 @@ import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls
*/
public class ChunkOutputStream extends OutputStream {
- private final BlockID blockID;
+ private BlockID blockID;
private final String key;
private final String traceID;
private final BlockData.Builder containerBlockData;
@@ -67,7 +68,6 @@ public class ChunkOutputStream extends OutputStream {
private final String streamId;
private int chunkIndex;
private int chunkSize;
- private long blockCommitSequenceId;
/**
* Creates a new ChunkOutputStream.
@@ -96,15 +96,14 @@ public class ChunkOutputStream extends OutputStream {
this.buffer = ByteBuffer.allocate(chunkSize);
this.streamId = UUID.randomUUID().toString();
this.chunkIndex = 0;
- blockCommitSequenceId = 0;
}
public ByteBuffer getBuffer() {
return buffer;
}
- public long getBlockCommitSequenceId() {
- return blockCommitSequenceId;
+ public BlockID getBlockID() {
+ return blockID;
}
@Override
@@ -165,8 +164,12 @@ public class ChunkOutputStream extends OutputStream {
try {
ContainerProtos.PutBlockResponseProto responseProto =
putBlock(xceiverClient, containerBlockData.build(), traceID);
- blockCommitSequenceId =
- responseProto.getCommittedBlockLength().getBlockCommitSequenceId();
+ BlockID responseBlockID = BlockID.getFromProtobuf(
+ responseProto.getCommittedBlockLength().getBlockID());
+ Preconditions.checkState(blockID.getContainerBlockID()
+ .equals(responseBlockID.getContainerBlockID()));
+ // updates the bcsId of the block
+ blockID = responseBlockID;
} catch (IOException e) {
throw new IOException(
"Unexpected Storage Container Exception: " + e.toString(), e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java
index 8149740..a863437 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java
@@ -23,52 +23,88 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.util.Objects;
/**
- * BlockID of ozone (containerID localID).
+ * BlockID of Ozone (containerID + localID + blockCommitSequenceId).
*/
+
public class BlockID {
- private long containerID;
- private long localID;
+
+ private ContainerBlockID containerBlockID;
+ private long blockCommitSequenceId;
public BlockID(long containerID, long localID) {
- this.containerID = containerID;
- this.localID = localID;
+ this(containerID, localID, 0);
+ }
+
+ private BlockID(long containerID, long localID, long bcsID) {
+ containerBlockID = new ContainerBlockID(containerID, localID);
+ blockCommitSequenceId = bcsID;
+ }
+
+ public BlockID(ContainerBlockID containerBlockID) {
+ this(containerBlockID, 0);
+ }
+
+ private BlockID(ContainerBlockID containerBlockID, long bcsId) {
+ this.containerBlockID = containerBlockID;
+ blockCommitSequenceId = bcsId;
}
public long getContainerID() {
- return containerID;
+ return containerBlockID.getContainerID();
}
public long getLocalID() {
- return localID;
+ return containerBlockID.getLocalID();
}
- @Override
- public String toString() {
- return new ToStringBuilder(this).
- append("containerID", containerID).
- append("localID", localID).
- toString();
+ public long getBlockCommitSequenceId() {
+ return blockCommitSequenceId;
}
- public HddsProtos.BlockID getProtobuf() {
- return HddsProtos.BlockID.newBuilder().
- setContainerID(containerID).setLocalID(localID).build();
+ public void setBlockCommitSequenceId(long blockCommitSequenceId) {
+ this.blockCommitSequenceId = blockCommitSequenceId;
}
- public static BlockID getFromProtobuf(HddsProtos.BlockID blockID) {
- return new BlockID(blockID.getContainerID(),
- blockID.getLocalID());
+ public ContainerBlockID getContainerBlockID() {
+ return containerBlockID;
+ }
+
+ public void setContainerBlockID(ContainerBlockID containerBlockID) {
+ this.containerBlockID = containerBlockID;
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this)
+ .append("containerID", containerBlockID.getContainerID())
+ .append("localID", containerBlockID.getLocalID())
+ .append("blockCommitSequenceId", blockCommitSequenceId)
+ .toString();
}
public ContainerProtos.DatanodeBlockID getDatanodeBlockIDProtobuf() {
return ContainerProtos.DatanodeBlockID.newBuilder().
- setContainerID(containerID).setLocalID(localID).build();
+ setContainerID(containerBlockID.getContainerID())
+ .setLocalID(containerBlockID.getLocalID())
+ .setBlockCommitSequenceId(blockCommitSequenceId).build();
}
public static BlockID getFromProtobuf(
ContainerProtos.DatanodeBlockID blockID) {
return new BlockID(blockID.getContainerID(),
- blockID.getLocalID());
+ blockID.getLocalID(), blockID.getBlockCommitSequenceId());
+ }
+
+ public HddsProtos.BlockID getProtobuf() {
+ return HddsProtos.BlockID.newBuilder()
+ .setContainerBlockID(containerBlockID.getProtobuf())
+ .setBlockCommitSequenceId(blockCommitSequenceId).build();
+ }
+
+ public static BlockID getFromProtobuf(HddsProtos.BlockID blockID) {
+ return new BlockID(
+ ContainerBlockID.getFromProtobuf(blockID.getContainerBlockID()),
+ blockID.getBlockCommitSequenceId());
}
@Override
@@ -80,11 +116,14 @@ public class BlockID {
return false;
}
BlockID blockID = (BlockID) o;
- return containerID == blockID.containerID && localID == blockID.localID;
+ return containerBlockID.equals(blockID.getContainerBlockID())
+ && blockCommitSequenceId == blockID.getBlockCommitSequenceId();
}
@Override
public int hashCode() {
- return Objects.hash(containerID, localID);
+ return Objects
+ .hash(containerBlockID.getContainerID(), containerBlockID.getLocalID(),
+ blockCommitSequenceId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java
new file mode 100644
index 0000000..82084f2
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.client;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+
+import java.util.Objects;
+
+/**
+ * BlockID returned by SCM during allocation of block (containerID + localID).
+ */
+public class ContainerBlockID {
+ private long containerID;
+ private long localID;
+
+ public ContainerBlockID(long containerID, long localID) {
+ this.containerID = containerID;
+ this.localID = localID;
+ }
+
+ public long getContainerID() {
+ return containerID;
+ }
+
+ public long getLocalID() {
+ return localID;
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this).
+ append("containerID", containerID).
+ append("localID", localID).
+ toString();
+ }
+
+ public HddsProtos.ContainerBlockID getProtobuf() {
+ return HddsProtos.ContainerBlockID.newBuilder().
+ setContainerID(containerID).setLocalID(localID).build();
+ }
+
+ public static ContainerBlockID getFromProtobuf(
+ HddsProtos.ContainerBlockID containerBlockID) {
+ return new ContainerBlockID(containerBlockID.getContainerID(),
+ containerBlockID.getLocalID());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ContainerBlockID blockID = (ContainerBlockID) o;
+ return containerID == blockID.containerID && localID == blockID.localID;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(containerID, localID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java
index f657b74..93af56d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.hdds.scm.container.common.helpers;
+import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.client.BlockID;
/**
* Allocated block wraps the result returned from SCM#allocateBlock which
@@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.client.BlockID;
*/
public final class AllocatedBlock {
private Pipeline pipeline;
- private BlockID blockID;
+ private ContainerBlockID containerBlockID;
// Indicates whether the client should create container before writing block.
private boolean shouldCreateContainer;
@@ -36,7 +36,7 @@ public final class AllocatedBlock {
*/
public static class Builder {
private Pipeline pipeline;
- private BlockID blockID;
+ private ContainerBlockID containerBlockID;
private boolean shouldCreateContainer;
public Builder setPipeline(Pipeline p) {
@@ -44,8 +44,8 @@ public final class AllocatedBlock {
return this;
}
- public Builder setBlockID(BlockID blockId) {
- this.blockID = blockId;
+ public Builder setContainerBlockID(ContainerBlockID blockId) {
+ this.containerBlockID = blockId;
return this;
}
@@ -55,14 +55,15 @@ public final class AllocatedBlock {
}
public AllocatedBlock build() {
- return new AllocatedBlock(pipeline, blockID, shouldCreateContainer);
+ return new AllocatedBlock(pipeline, containerBlockID,
+ shouldCreateContainer);
}
}
- private AllocatedBlock(Pipeline pipeline, BlockID blockID,
+ private AllocatedBlock(Pipeline pipeline, ContainerBlockID containerBlockID,
boolean shouldCreateContainer) {
this.pipeline = pipeline;
- this.blockID = blockID;
+ this.containerBlockID = containerBlockID;
this.shouldCreateContainer = shouldCreateContainer;
}
@@ -70,8 +71,8 @@ public final class AllocatedBlock {
return pipeline;
}
- public BlockID getBlockID() {
- return blockID;
+ public ContainerBlockID getBlockID() {
+ return containerBlockID;
}
public boolean getCreateContainer() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
index e684ae3..f868209 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
@@ -20,7 +20,7 @@ import com.google.common.base.Preconditions;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -102,7 +102,8 @@ public final class ScmBlockLocationProtocolClientSideTranslatorPB
response.getErrorMessage() : "Allocate block failed.");
}
AllocatedBlock.Builder builder = new AllocatedBlock.Builder()
- .setBlockID(BlockID.getFromProtobuf(response.getBlockID()))
+ .setContainerBlockID(
+ ContainerBlockID.getFromProtobuf(response.getContainerBlockID()))
.setPipeline(Pipeline.getFromProtobuf(response.getPipeline()))
.setShouldCreateContainer(response.getCreateContainer());
return builder.build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
index df1467b..150b1d6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
@@ -81,17 +81,14 @@ public final class ContainerProtocolCalls {
* @param xceiverClient client to perform call
* @param datanodeBlockID blockID to identify container
* @param traceID container protocol call args
- * @param blockCommitSequenceId latest commit Id of the block
* @return container protocol get block response
* @throws IOException if there is an I/O error while performing the call
*/
public static GetBlockResponseProto getBlock(XceiverClientSpi xceiverClient,
- DatanodeBlockID datanodeBlockID, String traceID,
- long blockCommitSequenceId) throws IOException {
+ DatanodeBlockID datanodeBlockID, String traceID) throws IOException {
GetBlockRequestProto.Builder readBlockRequest = GetBlockRequestProto
.newBuilder()
- .setBlockID(datanodeBlockID)
- .setBlockCommitSequenceId(blockCommitSequenceId);
+ .setBlockID(datanodeBlockID);
String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
ContainerCommandRequestProto request = ContainerCommandRequestProto
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
index 7a5403f..2f6030f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
@@ -61,7 +61,8 @@ public final class BlockGroup {
public static BlockGroup getFromProto(KeyBlocks proto) {
List<BlockID> blockIDs = new ArrayList<>();
for (HddsProtos.BlockID block : proto.getBlocksList()) {
- blockIDs.add(new BlockID(block.getContainerID(), block.getLocalID()));
+ blockIDs.add(new BlockID(block.getContainerBlockID().getContainerID(),
+ block.getContainerBlockID().getLocalID()));
}
return BlockGroup.newBuilder().setKeyName(proto.getKey())
.addAllBlockIDs(blockIDs).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
index 87cf824..3b9e57c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
@@ -34,7 +34,6 @@ import java.util.ArrayList;
public class BlockData {
private final BlockID blockID;
private final Map<String, String> metadata;
- private long blockCommitSequenceId;
/**
* Represent a list of chunks.
@@ -65,15 +64,14 @@ public class BlockData {
this.blockID = blockID;
this.metadata = new TreeMap<>();
this.size = 0;
- blockCommitSequenceId = 0;
}
public long getBlockCommitSequenceId() {
- return blockCommitSequenceId;
+ return blockID.getBlockCommitSequenceId();
}
public void setBlockCommitSequenceId(long blockCommitSequenceId) {
- this.blockCommitSequenceId = blockCommitSequenceId;
+ this.blockID.setBlockCommitSequenceId(blockCommitSequenceId);
}
/**
@@ -95,7 +93,6 @@ public class BlockData {
if (data.hasSize()) {
Preconditions.checkArgument(data.getSize() == blockData.getSize());
}
- blockData.setBlockCommitSequenceId(data.getBlockCommitSequenceId());
return blockData;
}
@@ -115,7 +112,6 @@ public class BlockData {
}
builder.addAllChunks(getChunks());
builder.setSize(size);
- builder.setBlockCommitSequenceId(blockCommitSequenceId);
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
index 37a1309..2ecf1f4 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
@@ -76,7 +76,7 @@ public final class ScmBlockLocationProtocolServerSideTranslatorPB
if (allocatedBlock != null) {
return
AllocateScmBlockResponseProto.newBuilder()
- .setBlockID(allocatedBlock.getBlockID().getProtobuf())
+ .setContainerBlockID(allocatedBlock.getBlockID().getProtobuf())
.setPipeline(allocatedBlock.getPipeline().getProtobufMessage())
.setCreateContainer(allocatedBlock.getCreateContainer())
.setErrorCode(AllocateScmBlockResponseProto.Error.success)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index f9262ba..318ec09 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -150,6 +150,7 @@ enum Result {
message DatanodeBlockID {
required int64 containerID = 1;
required int64 localID = 2;
+ optional uint64 blockCommitSequenceId = 3 [default = 0];
}
message KeyValue {
@@ -303,7 +304,6 @@ message BlockData {
repeated KeyValue metadata = 3;
repeated ChunkInfo chunks = 4;
optional int64 size = 5;
- optional uint64 blockCommitSequenceId = 6;
}
// Block Messages.
@@ -317,7 +317,6 @@ message PutBlockResponseProto {
message GetBlockRequestProto {
required DatanodeBlockID blockID = 1;
- optional uint64 blockCommitSequenceId = 2 [default = 0];
}
message GetBlockResponseProto {
@@ -336,7 +335,6 @@ message GetCommittedBlockLengthRequestProto {
message GetCommittedBlockLengthResponseProto {
required DatanodeBlockID blockID = 1;
required int64 blockLength = 2;
- optional uint64 blockCommitSequenceId = 3 [default = 0];
}
message DeleteBlockResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
index 01a0dde..dc68481 100644
--- a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
@@ -103,7 +103,7 @@ message AllocateScmBlockResponseProto {
unknownFailure = 4;
}
required Error errorCode = 1;
- optional BlockID blockID = 2;
+ optional ContainerBlockID containerBlockID = 2;
optional hadoop.hdds.Pipeline pipeline = 3;
optional bool createContainer = 4;
optional string errorMessage = 5;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index 9e813af..62b4833 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -186,7 +186,12 @@ enum ScmOps {
/**
* Block ID that uniquely identify a block by SCM.
*/
-message BlockID {
+message ContainerBlockID {
required int64 containerID = 1;
required int64 localID = 2;
}
+
+message BlockID {
+ required ContainerBlockID containerBlockID = 1;
+ optional uint64 blockCommitSequenceId = 2 [default = 0];
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index de72d25..b0bc08b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -483,8 +483,7 @@ public class KeyValueHandler extends Handler {
try {
BlockID blockID = BlockID.getFromProtobuf(
request.getGetBlock().getBlockID());
- responseData = blockManager.getBlock(kvContainer, blockID,
- request.getGetBlock().getBlockCommitSequenceId());
+ responseData = blockManager.getBlock(kvContainer, blockID);
long numBytes = responseData.getProtoBufMessage().toByteArray().length;
metrics.incContainerBytesStats(Type.GetBlock, numBytes);
@@ -759,7 +758,7 @@ public class KeyValueHandler extends Handler {
.getBlockID());
// TODO: add bcsId as a part of getSmallFile transaction
// by default its 0
- BlockData responseData = blockManager.getBlock(kvContainer, blockID, 0);
+ BlockData responseData = blockManager.getBlock(kvContainer, blockID);
ContainerProtos.ChunkInfo chunkInfo = null;
ByteString dataBuf = ByteString.EMPTY;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
index d6cadc8..e085fb0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
@@ -139,8 +139,6 @@ public final class BlockUtils {
committedBlockLengthResponseBuilder =
getCommittedBlockLengthResponseBuilder(blockLength,
blockData.getBlockID());
- committedBlockLengthResponseBuilder
- .setBlockCommitSequenceId(blockData.getBlockCommitSequenceId());
PutBlockResponseProto.Builder putKeyResponse =
PutBlockResponseProto.newBuilder();
putKeyResponse
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
index 3176189..e2e5700 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
@@ -129,13 +129,13 @@ public class BlockManagerImpl implements BlockManager {
*
* @param container - Container from which block need to be fetched.
* @param blockID - BlockID of the block.
- * @param bcsId latest commit Id of the block
* @return Key Data.
* @throws IOException
*/
@Override
- public BlockData getBlock(Container container, BlockID blockID, long bcsId)
+ public BlockData getBlock(Container container, BlockID blockID)
throws IOException {
+ long bcsId = blockID.getBlockCommitSequenceId();
Preconditions.checkNotNull(blockID,
"BlockID cannot be null in GetBlock request");
Preconditions.checkNotNull(blockID.getContainerID(),
@@ -162,7 +162,7 @@ public class BlockManagerImpl implements BlockManager {
}
ContainerProtos.BlockData blockData =
ContainerProtos.BlockData.parseFrom(kData);
- long id = blockData.getBlockCommitSequenceId();
+ long id = blockData.getBlockID().getBlockCommitSequenceId();
if (id < bcsId) {
throw new StorageContainerException(
"bcsId " + bcsId + " mismatches with existing block Id "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java
index 8c86583..6812b0d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java
@@ -45,11 +45,10 @@ public interface BlockManager {
*
* @param container - Container from which block need to be get.
* @param blockID - BlockID of the Block.
- * @param bcsId latest commit id of the block
* @return Block Data.
* @throws IOException
*/
- BlockData getBlock(Container container, BlockID blockID, long bcsId)
+ BlockData getBlock(Container container, BlockID blockID)
throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
index 65477d8..6fe6d81 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
@@ -113,7 +113,7 @@ public class TestBlockManagerImpl {
assertEquals(1, keyValueContainer.getContainerData().getKeyCount());
//Get Block
BlockData fromGetBlockData = blockManager.getBlock(keyValueContainer,
- blockData.getBlockID(), 0);
+ blockData.getBlockID());
assertEquals(blockData.getContainerID(), fromGetBlockData.getContainerID());
assertEquals(blockData.getLocalID(), fromGetBlockData.getLocalID());
@@ -139,7 +139,7 @@ public class TestBlockManagerImpl {
assertEquals(0,
keyValueContainer.getContainerData().getKeyCount());
try {
- blockManager.getBlock(keyValueContainer, blockID, 0);
+ blockManager.getBlock(keyValueContainer, blockID);
fail("testDeleteBlock");
} catch (StorageContainerException ex) {
GenericTestUtils.assertExceptionContains(
@@ -197,7 +197,7 @@ public class TestBlockManagerImpl {
keyValueContainer.getContainerData().getKeyCount());
try {
//Since the block has been deleted, we should not be able to find it
- blockManager.getBlock(keyValueContainer, blockID, 0);
+ blockManager.getBlock(keyValueContainer, blockID);
fail("testGetNoSuchBlock failed");
} catch (StorageContainerException ex) {
GenericTestUtils.assertExceptionContains(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 681d021..c878d97 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.block;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmUtils;
@@ -318,7 +319,7 @@ public class BlockManagerImpl implements EventHandler<Boolean>,
AllocatedBlock.Builder abb =
new AllocatedBlock.Builder()
- .setBlockID(new BlockID(containerID, localID))
+ .setContainerBlockID(new ContainerBlockID(containerID, localID))
.setPipeline(containerWithPipeline.getPipeline())
.setShouldCreateContainer(createContainer);
LOG.trace("New block allocated : {} Container ID: {}", localID,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
index 0c09fc8..5632756 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
@@ -297,8 +297,7 @@ public class ChunkGroupInputStream extends InputStream implements Seekable {
ContainerProtos.DatanodeBlockID datanodeBlockID = blockID
.getDatanodeBlockIDProtobuf();
ContainerProtos.GetBlockResponseProto response = ContainerProtocolCalls
- .getBlock(xceiverClient, datanodeBlockID, requestId,
- omKeyLocationInfo.getBlockCommitSequenceId());
+ .getBlock(xceiverClient, datanodeBlockID, requestId);
List<ContainerProtos.ChunkInfo> chunks =
response.getBlockData().getChunksList();
for (ContainerProtos.ChunkInfo chunk : chunks) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index 6d13bb2..78d69c1 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -128,7 +128,6 @@ public class ChunkGroupOutputStream extends OutputStream {
new OmKeyLocationInfo.Builder().setBlockID(streamEntry.blockID)
.setShouldCreateContainer(false)
.setLength(streamEntry.currentPosition).setOffset(0)
- .setBlockCommitSequenceId(streamEntry.getBlockCommitSequenceId())
.build();
locationInfoList.add(info);
}
@@ -614,7 +613,7 @@ public class ChunkGroupOutputStream extends OutputStream {
private static class ChunkOutputStreamEntry extends OutputStream {
private OutputStream outputStream;
- private final BlockID blockID;
+ private BlockID blockID;
private final String key;
private final XceiverClientManager xceiverClientManager;
private final XceiverClientSpi xceiverClient;
@@ -700,6 +699,11 @@ public class ChunkGroupOutputStream extends OutputStream {
public void close() throws IOException {
if (this.outputStream != null) {
this.outputStream.close();
+ // after closing the chunkOutPutStream, blockId would have been
+ // reconstructed with updated bcsId
+ if (this.outputStream instanceof ChunkOutputStream) {
+ this.blockID = ((ChunkOutputStream) outputStream).getBlockID();
+ }
}
}
@@ -711,19 +715,6 @@ public class ChunkGroupOutputStream extends OutputStream {
throw new IOException("Invalid Output Stream for Key: " + key);
}
- long getBlockCommitSequenceId() throws IOException {
- if (this.outputStream instanceof ChunkOutputStream) {
- ChunkOutputStream out = (ChunkOutputStream) this.outputStream;
- return out.getBlockCommitSequenceId();
- } else if (outputStream == null) {
- // For a pre allocated block for which no write has been initiated,
- // the OutputStream will be null here.
- // In such cases, the default blockCommitSequenceId will be 0
- return 0;
- }
- throw new IOException("Invalid Output Stream for Key: " + key);
- }
-
public void cleanup() {
checkStream();
if (this.outputStream instanceof ChunkOutputStream) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java
index 9d54cea..d86153d 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java
@@ -31,15 +31,13 @@ public final class OmKeyLocationInfo {
private final long offset;
// the version number indicating when this block was added
private long createVersion;
- private final long blockCommitSequenceId;
private OmKeyLocationInfo(BlockID blockID, boolean shouldCreateContainer,
- long length, long offset, long blockCommitSequenceId) {
+ long length, long offset) {
this.blockID = blockID;
this.shouldCreateContainer = shouldCreateContainer;
this.length = length;
this.offset = offset;
- this.blockCommitSequenceId = blockCommitSequenceId;
}
public void setCreateVersion(long version) {
@@ -79,7 +77,7 @@ public final class OmKeyLocationInfo {
}
public long getBlockCommitSequenceId() {
- return blockCommitSequenceId;
+ return blockID.getBlockCommitSequenceId();
}
/**
@@ -90,7 +88,6 @@ public final class OmKeyLocationInfo {
private boolean shouldCreateContainer;
private long length;
private long offset;
- private long blockCommitSequenceId;
public Builder setBlockID(BlockID blockId) {
this.blockID = blockId;
@@ -112,14 +109,9 @@ public final class OmKeyLocationInfo {
return this;
}
- public Builder setBlockCommitSequenceId(long sequenceId) {
- this.blockCommitSequenceId = sequenceId;
- return this;
- }
-
public OmKeyLocationInfo build() {
return new OmKeyLocationInfo(blockID,
- shouldCreateContainer, length, offset, blockCommitSequenceId);
+ shouldCreateContainer, length, offset);
}
}
@@ -130,7 +122,6 @@ public final class OmKeyLocationInfo {
.setLength(length)
.setOffset(offset)
.setCreateVersion(createVersion)
- .setBlockCommitSequenceId(blockCommitSequenceId)
.build();
}
@@ -139,8 +130,7 @@ public final class OmKeyLocationInfo {
BlockID.getFromProtobuf(keyLocation.getBlockID()),
keyLocation.getShouldCreateContainer(),
keyLocation.getLength(),
- keyLocation.getOffset(),
- keyLocation.getBlockCommitSequenceId());
+ keyLocation.getOffset());
info.setCreateVersion(keyLocation.getCreateVersion());
return info;
}
@@ -152,7 +142,6 @@ public final class OmKeyLocationInfo {
", shouldCreateContainer=" + shouldCreateContainer +
", length=" + length +
", offset=" + offset +
- ", blockCommitSequenceId=" + blockCommitSequenceId +
", createVersion=" + createVersion + '}';
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index d36cace..8c4c409 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -252,7 +252,6 @@ message KeyLocation {
required uint64 length = 4;
// indicated at which version this block gets created.
optional uint64 createVersion = 5;
- optional uint64 blockCommitSequenceId = 6;
}
message KeyLocationList {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
index 1789e55..5153b41 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
@@ -151,7 +151,7 @@ public class TestContainerReplication {
.getHandler(ContainerType.KeyValueContainer);
BlockData key = handler.getBlockManager()
- .getBlock(container, BlockID.getFromProtobuf(blockID), 0);
+ .getBlock(container, BlockID.getFromProtobuf(blockID));
Assert.assertNotNull(key);
Assert.assertEquals(1, key.getChunks().size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
index 360b683..0ae63e3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
@@ -256,6 +256,6 @@ public class TestCloseContainerHandler {
openContainerBlockMap.getBlockDataMap(testContainerID));
// Make sure the key got committed
Assert.assertNotNull(handler.getBlockManager()
- .getBlock(container, blockID, 0));
+ .getBlock(container, blockID));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 293aac8..f81ee57 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -558,7 +558,7 @@ public class TestContainerPersistence {
blockData.setChunks(chunkList);
blockManager.putBlock(container, blockData);
BlockData readBlockData = blockManager.
- getBlock(container, blockData.getBlockID(), 0);
+ getBlock(container, blockData.getBlockID());
ChunkInfo readChunk =
ChunkInfo.getFromProtoBuf(readBlockData.getChunks().get(0));
Assert.assertEquals(info.getChecksum(), readChunk.getChecksum());
@@ -596,25 +596,27 @@ public class TestContainerPersistence {
blockManager.putBlock(container, blockData);
BlockData readBlockData;
try {
+ blockID1.setBlockCommitSequenceId(5);
// read with bcsId higher than container bcsId
blockManager.
- getBlock(container, blockID1, 5);
+ getBlock(container, blockID1);
Assert.fail("Expected exception not thrown");
} catch (StorageContainerException sce) {
Assert.assertTrue(sce.getResult() == UNKNOWN_BCSID);
}
try {
+ blockID1.setBlockCommitSequenceId(4);
// read with bcsId lower than container bcsId but greater than committed
// bcsId.
blockManager.
- getBlock(container, blockID1, 4);
+ getBlock(container, blockID1);
Assert.fail("Expected exception not thrown");
} catch (StorageContainerException sce) {
Assert.assertTrue(sce.getResult() == BCSID_MISMATCH);
}
readBlockData = blockManager.
- getBlock(container, blockData.getBlockID(), 4);
+ getBlock(container, blockData.getBlockID());
ChunkInfo readChunk =
ChunkInfo.getFromProtoBuf(readBlockData.getChunks().get(0));
Assert.assertEquals(info.getChecksum(), readChunk.getChecksum());
@@ -666,7 +668,7 @@ public class TestContainerPersistence {
blockData.setChunks(chunkProtoList);
blockManager.putBlock(container, blockData);
BlockData readBlockData = blockManager.
- getBlock(container, blockData.getBlockID(), 0);
+ getBlock(container, blockData.getBlockID());
ChunkInfo lastChunk = chunkList.get(chunkList.size() - 1);
ChunkInfo readChunk =
ChunkInfo.getFromProtoBuf(readBlockData.getChunks().get(readBlockData
@@ -694,7 +696,7 @@ public class TestContainerPersistence {
blockManager.deleteBlock(container, blockID);
exception.expect(StorageContainerException.class);
exception.expectMessage("Unable to find the block.");
- blockManager.getBlock(container, blockData.getBlockID(), 0);
+ blockManager.getBlock(container, blockData.getBlockID());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
index 834dff0..974bb97 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
@@ -239,16 +239,18 @@ public class TestGetCommittedBlockLengthAndPutKey {
ContainerTestHelper
.getPutBlockRequest(pipeline, writeChunkRequest.getWriteChunk());
response = client.sendCommand(putKeyRequest).getPutBlock();
+ Assert.assertEquals(
+ response.getCommittedBlockLength().getBlockLength(), data.length);
+ Assert.assertTrue(response.getCommittedBlockLength().getBlockID()
+ .getBlockCommitSequenceId() > 0);
+ BlockID responseBlockID = BlockID
+ .getFromProtobuf(response.getCommittedBlockLength().getBlockID());
+ blockID
+ .setBlockCommitSequenceId(responseBlockID.getBlockCommitSequenceId());
// make sure the block ids in the request and response are same.
// This will also ensure that closing the container committed the block
// on the Datanodes.
- Assert.assertEquals(BlockID
- .getFromProtobuf(response.getCommittedBlockLength().getBlockID()),
- blockID);
- Assert.assertEquals(
- response.getCommittedBlockLength().getBlockLength(), data.length);
- Assert.assertTrue(
- response.getCommittedBlockLength().getBlockCommitSequenceId() > 0);
+ Assert.assertEquals(responseBlockID, blockID);
xceiverClientManager.releaseClient(client);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 08905eb..d83d9a3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -700,7 +700,7 @@ public class TestKeys {
KeyValueContainer container = (KeyValueContainer) cm.getContainerSet()
.getContainer(location.getBlockID().getContainerID());
BlockData blockInfo = keyValueHandler.getBlockManager()
- .getBlock(container, location.getBlockID(), 0);
+ .getBlock(container, location.getBlockID());
KeyValueContainerData containerData =
(KeyValueContainerData) container.getContainerData();
File dataDir = new File(containerData.getChunksPath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 340197f..733ed85 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
@@ -169,10 +170,9 @@ public class KeyManagerImpl implements KeyManager {
throw ex;
}
OmKeyLocationInfo info = new OmKeyLocationInfo.Builder()
- .setBlockID(allocatedBlock.getBlockID())
+ .setBlockID(new BlockID(allocatedBlock.getBlockID()))
.setShouldCreateContainer(allocatedBlock.getCreateContainer())
.setLength(scmBlockSize)
- .setBlockCommitSequenceId(0)
.setOffset(0)
.build();
// current version not committed, so new blocks coming now are added to
@@ -234,10 +234,9 @@ public class KeyManagerImpl implements KeyManager {
throw ex;
}
OmKeyLocationInfo subKeyInfo = new OmKeyLocationInfo.Builder()
- .setBlockID(allocatedBlock.getBlockID())
+ .setBlockID(new BlockID(allocatedBlock.getBlockID()))
.setShouldCreateContainer(allocatedBlock.getCreateContainer())
.setLength(allocateSize)
- .setBlockCommitSequenceId(0)
.setOffset(0)
.build();
locations.add(subKeyInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7757331d/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java
index 259f842..2076ced 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmInfo;
@@ -120,7 +121,7 @@ public class ScmBlockLocationTestIngClient implements ScmBlockLocationProtocol {
long localID = Time.monotonicNow();
AllocatedBlock.Builder abb =
new AllocatedBlock.Builder()
- .setBlockID(new BlockID(containerID, localID))
+ .setContainerBlockID(new ContainerBlockID(containerID, localID))
.setPipeline(pipeline)
.setShouldCreateContainer(false);
return abb.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-8854. Upgrade jquery
datatable version references to v1.10.19. Contributed by Akhil PB.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/js/jquery.dataTables.min.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/js/jquery.dataTables.min.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/js/jquery.dataTables.min.js
deleted file mode 100644
index 85dd817..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/js/jquery.dataTables.min.js
+++ /dev/null
@@ -1,160 +0,0 @@
-/*! DataTables 1.10.7
- * ©2008-2015 SpryMedia Ltd - datatables.net/license
- */
-(function(Ea,Q,k){var P=function(h){function W(a){var b,c,e={};h.each(a,function(d){if((b=d.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=d.replace(b[0],b[2].toLowerCase()),e[c]=d,"o"===b[1]&&W(a[d])});a._hungarianMap=e}function H(a,b,c){a._hungarianMap||W(a);var e;h.each(b,function(d){e=a._hungarianMap[d];if(e!==k&&(c||b[e]===k))"o"===e.charAt(0)?(b[e]||(b[e]={}),h.extend(!0,b[e],b[d]),H(a[e],b[e],c)):b[e]=b[d]})}function P(a){var b=m.defaults.oLanguage,c=a.sZeroRecords;
-!a.sEmptyTable&&(c&&"No data available in table"===b.sEmptyTable)&&E(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(c&&"Loading..."===b.sLoadingRecords)&&E(a,a,"sZeroRecords","sLoadingRecords");a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&db(a)}function eb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate");
-A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");if(a=a.aoSearchCols)for(var b=0,c=a.length;b<c;b++)a[b]&&H(m.models.oSearch,a[b])}function fb(a){A(a,"orderable","bSortable");A(a,"orderData","aDataSort");A(a,"orderSequence","asSorting");A(a,"orderDataType","sortDataType");var b=a.aDataSort;b&&!h.isArray(b)&&(a.aDataSort=[b])}function gb(a){var a=a.oBrowser,b=h("<div/>").css({position:"absolute",top:0,left:0,height:1,width:1,overflow:"hidden"}).append(h("<div/>").css({position:"absolute",
-top:1,left:1,width:100,overflow:"scroll"}).append(h('<div class="test"/>').css({width:"100%",height:10}))).appendTo("body"),c=b.find(".test");a.bScrollOversize=100===c[0].offsetWidth;a.bScrollbarLeft=1!==Math.round(c.offset().left);b.remove()}function hb(a,b,c,e,d,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;e!==d;)a.hasOwnProperty(e)&&(g=j?b(g,a[e],e,a):a[e],j=!0,e+=f);return g}function Fa(a,b){var c=m.defaults.column,e=a.aoColumns.length,c=h.extend({},m.models.oColumn,c,{nTh:b?b:Q.createElement("th"),sTitle:c.sTitle?
-c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[e],mData:c.mData?c.mData:e,idx:e});a.aoColumns.push(c);c=a.aoPreSearchCols;c[e]=h.extend({},m.models.oSearch,c[e]);ka(a,e,h(b).data())}function ka(a,b,c){var b=a.aoColumns[b],e=a.oClasses,d=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=d.attr("width")||null;var f=(d.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(fb(c),H(m.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&&
-(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),h.extend(b,c),E(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),E(b,c,"aDataSort"));var g=b.mData,j=R(g),i=b.mRender?R(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};b._bAttrSrc=h.isPlainObject(g)&&(c(g.sort)||c(g.type)||c(g.filter));b.fnGetData=function(a,b,c){var e=j(a,b,k,c);return i&&b?i(e,b,a,c):e};b.fnSetData=function(a,b,c){return S(g)(a,b,c)};"number"!==typeof g&&
-(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,d.addClass(e.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=e.sSortableNone,b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=e.sSortableAsc,b.sSortingClassJUI=e.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=e.sSortableDesc,b.sSortingClassJUI=e.sSortJUIDescAllowed):(b.sSortingClass=e.sSortable,b.sSortingClassJUI=e.sSortJUI)}function X(a){if(!1!==a.oFeatures.bAutoWidth){var b=
-a.aoColumns;Ga(a);for(var c=0,e=b.length;c<e;c++)b[c].nTh.style.width=b[c].sWidth}b=a.oScroll;(""!==b.sY||""!==b.sX)&&Y(a);w(a,null,"column-sizing",[a])}function la(a,b){var c=Z(a,"bVisible");return"number"===typeof c[b]?c[b]:null}function $(a,b){var c=Z(a,"bVisible"),c=h.inArray(b,c);return-1!==c?c:null}function aa(a){return Z(a,"bVisible").length}function Z(a,b){var c=[];h.map(a.aoColumns,function(a,d){a[b]&&c.push(d)});return c}function Ha(a){var b=a.aoColumns,c=a.aoData,e=m.ext.type.detect,d,
-f,g,j,i,h,l,q,n;d=0;for(f=b.length;d<f;d++)if(l=b[d],n=[],!l.sType&&l._sManualType)l.sType=l._sManualType;else if(!l.sType){g=0;for(j=e.length;g<j;g++){i=0;for(h=c.length;i<h;i++){n[i]===k&&(n[i]=x(a,i,d,"type"));q=e[g](n[i],a);if(!q&&g!==e.length-1)break;if("html"===q)break}if(q){l.sType=q;break}}l.sType||(l.sType="string")}}function ib(a,b,c,e){var d,f,g,j,i,o,l=a.aoColumns;if(b)for(d=b.length-1;0<=d;d--){o=b[d];var q=o.targets!==k?o.targets:o.aTargets;h.isArray(q)||(q=[q]);f=0;for(g=q.length;f<
-g;f++)if("number"===typeof q[f]&&0<=q[f]){for(;l.length<=q[f];)Fa(a);e(q[f],o)}else if("number"===typeof q[f]&&0>q[f])e(l.length+q[f],o);else if("string"===typeof q[f]){j=0;for(i=l.length;j<i;j++)("_all"==q[f]||h(l[j].nTh).hasClass(q[f]))&&e(j,o)}}if(c){d=0;for(a=c.length;d<a;d++)e(d,c[d])}}function K(a,b,c,e){var d=a.aoData.length,f=h.extend(!0,{},m.models.oRow,{src:c?"dom":"data"});f._aData=b;a.aoData.push(f);for(var b=a.aoColumns,f=0,g=b.length;f<g;f++)c&&Ia(a,d,f,x(a,d,f)),b[f].sType=null;a.aiDisplayMaster.push(d);
-(c||!a.oFeatures.bDeferRender)&&Ja(a,d,c,e);return d}function ma(a,b){var c;b instanceof h||(b=h(b));return b.map(function(b,d){c=na(a,d);return K(a,c.data,d,c.cells)})}function x(a,b,c,e){var d=a.iDraw,f=a.aoColumns[c],g=a.aoData[b]._aData,j=f.sDefaultContent,c=f.fnGetData(g,e,{settings:a,row:b,col:c});if(c===k)return a.iDrawError!=d&&null===j&&(I(a,0,"Requested unknown parameter "+("function"==typeof f.mData?"{function}":"'"+f.mData+"'")+" for row "+b,4),a.iDrawError=d),j;if((c===g||null===c)&&
-null!==j)c=j;else if("function"===typeof c)return c.call(g);return null===c&&"display"==e?"":c}function Ia(a,b,c,e){a.aoColumns[c].fnSetData(a.aoData[b]._aData,e,{settings:a,row:b,col:c})}function Ka(a){return h.map(a.match(/(\\.|[^\.])+/g),function(a){return a.replace(/\\./g,".")})}function R(a){if(h.isPlainObject(a)){var b={};h.each(a,function(a,c){c&&(b[a]=R(c))});return function(a,c,f,g){var j=b[c]||b._;return j!==k?j(a,c,f,g):a}}if(null===a)return function(a){return a};if("function"===typeof a)return function(b,
-c,f,g){return a(b,c,f,g)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var c=function(a,b,f){var g,j;if(""!==f){j=Ka(f);for(var i=0,h=j.length;i<h;i++){f=j[i].match(ba);g=j[i].match(T);if(f){j[i]=j[i].replace(ba,"");""!==j[i]&&(a=a[j[i]]);g=[];j.splice(0,i+1);j=j.join(".");i=0;for(h=a.length;i<h;i++)g.push(c(a[i],b,j));a=f[0].substring(1,f[0].length-1);a=""===a?g:g.join(a);break}else if(g){j[i]=j[i].replace(T,"");a=a[j[i]]();continue}if(null===a||a[j[i]]===
-k)return k;a=a[j[i]]}}return a};return function(b,d){return c(b,d,a)}}return function(b){return b[a]}}function S(a){if(h.isPlainObject(a))return S(a._);if(null===a)return function(){};if("function"===typeof a)return function(b,e,d){a(b,"set",e,d)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var b=function(a,e,d){var d=Ka(d),f;f=d[d.length-1];for(var g,j,i=0,h=d.length-1;i<h;i++){g=d[i].match(ba);j=d[i].match(T);if(g){d[i]=d[i].replace(ba,"");a[d[i]]=[];
-f=d.slice();f.splice(0,i+1);g=f.join(".");j=0;for(h=e.length;j<h;j++)f={},b(f,e[j],g),a[d[i]].push(f);return}j&&(d[i]=d[i].replace(T,""),a=a[d[i]](e));if(null===a[d[i]]||a[d[i]]===k)a[d[i]]={};a=a[d[i]]}if(f.match(T))a[f.replace(T,"")](e);else a[f.replace(ba,"")]=e};return function(c,e){return b(c,e,a)}}return function(b,e){b[a]=e}}function La(a){return D(a.aoData,"_aData")}function oa(a){a.aoData.length=0;a.aiDisplayMaster.length=0;a.aiDisplay.length=0}function pa(a,b,c){for(var e=-1,d=0,f=a.length;d<
-f;d++)a[d]==b?e=d:a[d]>b&&a[d]--; -1!=e&&c===k&&a.splice(e,1)}function ca(a,b,c,e){var d=a.aoData[b],f,g=function(c,f){for(;c.childNodes.length;)c.removeChild(c.firstChild);c.innerHTML=x(a,b,f,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===d.src)d._aData=na(a,d,e,e===k?k:d._aData).data;else{var j=d.anCells;if(j)if(e!==k)g(j[e],e);else{c=0;for(f=j.length;c<f;c++)g(j[c],c)}}d._aSortData=null;d._aFilterData=null;g=a.aoColumns;if(e!==k)g[e].sType=null;else{c=0;for(f=g.length;c<f;c++)g[c].sType=null;
-Ma(d)}}function na(a,b,c,e){var d=[],f=b.firstChild,g,j=0,i,o=a.aoColumns,l=a._rowReadObject,e=e||l?{}:[],q=function(a,b){if("string"===typeof a){var c=a.indexOf("@");-1!==c&&(c=a.substring(c+1),S(a)(e,b.getAttribute(c)))}},a=function(a){if(c===k||c===j)g=o[j],i=h.trim(a.innerHTML),g&&g._bAttrSrc?(S(g.mData._)(e,i),q(g.mData.sort,a),q(g.mData.type,a),q(g.mData.filter,a)):l?(g._setter||(g._setter=S(g.mData)),g._setter(e,i)):e[j]=i;j++};if(f)for(;f;){b=f.nodeName.toUpperCase();if("TD"==b||"TH"==b)a(f),
-d.push(f);f=f.nextSibling}else{d=b.anCells;f=0;for(b=d.length;f<b;f++)a(d[f])}return{data:e,cells:d}}function Ja(a,b,c,e){var d=a.aoData[b],f=d._aData,g=[],j,i,h,l,q;if(null===d.nTr){j=c||Q.createElement("tr");d.nTr=j;d.anCells=g;j._DT_RowIndex=b;Ma(d);l=0;for(q=a.aoColumns.length;l<q;l++){h=a.aoColumns[l];i=c?e[l]:Q.createElement(h.sCellType);g.push(i);if(!c||h.mRender||h.mData!==l)i.innerHTML=x(a,b,l,"display");h.sClass&&(i.className+=" "+h.sClass);h.bVisible&&!c?j.appendChild(i):!h.bVisible&&c&&
-i.parentNode.removeChild(i);h.fnCreatedCell&&h.fnCreatedCell.call(a.oInstance,i,x(a,b,l),f,b,l)}w(a,"aoRowCreatedCallback",null,[j,f,b])}d.nTr.setAttribute("role","row")}function Ma(a){var b=a.nTr,c=a._aData;if(b){c.DT_RowId&&(b.id=c.DT_RowId);if(c.DT_RowClass){var e=c.DT_RowClass.split(" ");a.__rowc=a.__rowc?Na(a.__rowc.concat(e)):e;h(b).removeClass(a.__rowc.join(" ")).addClass(c.DT_RowClass)}c.DT_RowAttr&&h(b).attr(c.DT_RowAttr);c.DT_RowData&&h(b).data(c.DT_RowData)}}function jb(a){var b,c,e,d,
-f,g=a.nTHead,j=a.nTFoot,i=0===h("th, td",g).length,o=a.oClasses,l=a.aoColumns;i&&(d=h("<tr/>").appendTo(g));b=0;for(c=l.length;b<c;b++)f=l[b],e=h(f.nTh).addClass(f.sClass),i&&e.appendTo(d),a.oFeatures.bSort&&(e.addClass(f.sSortingClass),!1!==f.bSortable&&(e.attr("tabindex",a.iTabIndex).attr("aria-controls",a.sTableId),Oa(a,f.nTh,b))),f.sTitle!=e.html()&&e.html(f.sTitle),Pa(a,"header")(a,e,f,o);i&&da(a.aoHeader,g);h(g).find(">tr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(o.sHeaderTH);
-h(j).find(">tr>th, >tr>td").addClass(o.sFooterTH);if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b<c;b++)f=l[b],f.nTf=a[b].cell,f.sClass&&h(f.nTf).addClass(f.sClass)}}function ea(a,b,c){var e,d,f,g=[],j=[],i=a.aoColumns.length,o;if(b){c===k&&(c=!1);e=0;for(d=b.length;e<d;e++){g[e]=b[e].slice();g[e].nTr=b[e].nTr;for(f=i-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[e].splice(f,1);j.push([])}e=0;for(d=g.length;e<d;e++){if(a=g[e].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[e].length;f<b;f++)if(o=
-i=1,j[e][f]===k){a.appendChild(g[e][f].cell);for(j[e][f]=1;g[e+i]!==k&&g[e][f].cell==g[e+i][f].cell;)j[e+i][f]=1,i++;for(;g[e][f+o]!==k&&g[e][f].cell==g[e][f+o].cell;){for(c=0;c<i;c++)j[e+c][f+o]=1;o++}h(g[e][f].cell).attr("rowspan",i).attr("colspan",o)}}}}function M(a){var b=w(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))C(a,!1);else{var b=[],c=0,e=a.asStripeClasses,d=e.length,f=a.oLanguage,g=a.iInitDisplayStart,j="ssp"==B(a),i=a.aiDisplay;a.bDrawing=!0;g!==k&&-1!==g&&(a._iDisplayStart=
-j?g:g>=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=-1);var g=a._iDisplayStart,o=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&!kb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:o;for(j=j?0:g;j<f;j++){var l=i[j],q=a.aoData[l];null===q.nTr&&Ja(a,l);l=q.nTr;if(0!==d){var n=e[c%d];q._sRowStripe!=n&&(h(l).removeClass(q._sRowStripe).addClass(n),q._sRowStripe=n)}w(a,"aoRowCallback",null,[l,q._aData,c,j]);b.push(l);c++}}else c=f.sZeroRecords,
-1==a.iDraw&&"ajax"==B(a)?c=f.sLoadingRecords:f.sEmptyTable&&0===a.fnRecordsTotal()&&(c=f.sEmptyTable),b[0]=h("<tr/>",{"class":d?e[0]:""}).append(h("<td />",{valign:"top",colSpan:aa(a),"class":a.oClasses.sRowEmpty}).html(c))[0];w(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],La(a),g,o,i]);w(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],La(a),g,o,i]);e=h(a.nTBody);e.children().detach();e.append(h(b));w(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=
-!1}}function N(a,b){var c=a.oFeatures,e=c.bFilter;c.bSort&&lb(a);e?fa(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;M(a);a._drawHold=!1}function mb(a){var b=a.oClasses,c=h(a.nTable),c=h("<div/>").insertBefore(c),e=a.oFeatures,d=h("<div/>",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=d[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,o,l,q,n=0;n<f.length;n++){g=
-null;j=f[n];if("<"==j){i=h("<div/>")[0];o=f[n+1];if("'"==o||'"'==o){l="";for(q=2;f[n+q]!=o;)l+=f[n+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(o=l.split("."),i.id=o[0].substr(1,o[0].length-1),i.className=o[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;n+=q}d.append(i);d=h(i)}else if(">"==j)d=d.parent();else if("l"==j&&e.bPaginate&&e.bLengthChange)g=nb(a);else if("f"==j&&e.bFilter)g=ob(a);else if("r"==j&&e.bProcessing)g=pb(a);else if("t"==j)g=qb(a);else if("i"==
-j&&e.bInfo)g=rb(a);else if("p"==j&&e.bPaginate)g=sb(a);else if(0!==m.ext.feature.length){i=m.ext.feature;q=0;for(o=i.length;q<o;q++)if(j==i[q].cFeature){g=i[q].fnInit(a);break}}g&&(i=a.aanFeatures,i[j]||(i[j]=[]),i[j].push(g),d.append(g))}c.replaceWith(d)}function da(a,b){var c=h(b).children("tr"),e,d,f,g,j,i,o,l,q,n;a.splice(0,a.length);f=0;for(i=c.length;f<i;f++)a.push([]);f=0;for(i=c.length;f<i;f++){e=c[f];for(d=e.firstChild;d;){if("TD"==d.nodeName.toUpperCase()||"TH"==d.nodeName.toUpperCase()){l=
-1*d.getAttribute("colspan");q=1*d.getAttribute("rowspan");l=!l||0===l||1===l?1:l;q=!q||0===q||1===q?1:q;g=0;for(j=a[f];j[g];)g++;o=g;n=1===l?!0:!1;for(j=0;j<l;j++)for(g=0;g<q;g++)a[f+g][o+j]={cell:d,unique:n},a[f+g].nTr=e}d=d.nextSibling}}}function qa(a,b,c){var e=[];c||(c=a.aoHeader,b&&(c=[],da(c,b)));for(var b=0,d=c.length;b<d;b++)for(var f=0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!e[f]||!a.bSortCellsTop))e[f]=c[b][f].cell;return e}function ra(a,b,c){w(a,"aoServerParams","serverParams",[b]);
-if(b&&h.isArray(b)){var e={},d=/(.*?)\[\]$/;h.each(b,function(a,b){var c=b.name.match(d);c?(c=c[0],e[c]||(e[c]=[]),e[c].push(b.value)):e[b.name]=b.value});b=e}var f,g=a.ajax,j=a.oInstance,i=function(b){w(a,null,"xhr",[a,b,a.jqXHR]);c(b)};if(h.isPlainObject(g)&&g.data){f=g.data;var o=h.isFunction(f)?f(b,a):f,b=h.isFunction(f)&&o?o:h.extend(!0,b,o);delete g.data}o={data:b,success:function(b){var c=b.error||b.sError;c&&I(a,0,c);a.json=b;i(b)},dataType:"json",cache:!1,type:a.sServerMethod,error:function(b,
-c){var f=w(a,null,"xhr",[a,null,a.jqXHR]);-1===h.inArray(!0,f)&&("parsererror"==c?I(a,0,"Invalid JSON response",1):4===b.readyState&&I(a,0,"Ajax error",7));C(a,!1)}};a.oAjaxData=b;w(a,null,"preXhr",[a,b]);a.fnServerData?a.fnServerData.call(j,a.sAjaxSource,h.map(b,function(a,b){return{name:b,value:a}}),i,a):a.sAjaxSource||"string"===typeof g?a.jqXHR=h.ajax(h.extend(o,{url:g||a.sAjaxSource})):h.isFunction(g)?a.jqXHR=g.call(j,b,i,a):(a.jqXHR=h.ajax(h.extend(o,g)),g.data=f)}function kb(a){return a.bAjaxDataGet?
-(a.iDraw++,C(a,!0),ra(a,tb(a),function(b){ub(a,b)}),!1):!0}function tb(a){var b=a.aoColumns,c=b.length,e=a.oFeatures,d=a.oPreviousSearch,f=a.aoPreSearchCols,g,j=[],i,o,l,q=U(a);g=a._iDisplayStart;i=!1!==e.bPaginate?a._iDisplayLength:-1;var n=function(a,b){j.push({name:a,value:b})};n("sEcho",a.iDraw);n("iColumns",c);n("sColumns",D(b,"sName").join(","));n("iDisplayStart",g);n("iDisplayLength",i);var k={draw:a.iDraw,columns:[],order:[],start:g,length:i,search:{value:d.sSearch,regex:d.bRegex}};for(g=
-0;g<c;g++)o=b[g],l=f[g],i="function"==typeof o.mData?"function":o.mData,k.columns.push({data:i,name:o.sName,searchable:o.bSearchable,orderable:o.bSortable,search:{value:l.sSearch,regex:l.bRegex}}),n("mDataProp_"+g,i),e.bFilter&&(n("sSearch_"+g,l.sSearch),n("bRegex_"+g,l.bRegex),n("bSearchable_"+g,o.bSearchable)),e.bSort&&n("bSortable_"+g,o.bSortable);e.bFilter&&(n("sSearch",d.sSearch),n("bRegex",d.bRegex));e.bSort&&(h.each(q,function(a,b){k.order.push({column:b.col,dir:b.dir});n("iSortCol_"+a,b.col);
-n("sSortDir_"+a,b.dir)}),n("iSortingCols",q.length));b=m.ext.legacy.ajax;return null===b?a.sAjaxSource?j:k:b?j:k}function ub(a,b){var c=sa(a,b),e=b.sEcho!==k?b.sEcho:b.draw,d=b.iTotalRecords!==k?b.iTotalRecords:b.recordsTotal,f=b.iTotalDisplayRecords!==k?b.iTotalDisplayRecords:b.recordsFiltered;if(e){if(1*e<a.iDraw)return;a.iDraw=1*e}oa(a);a._iRecordsTotal=parseInt(d,10);a._iRecordsDisplay=parseInt(f,10);e=0;for(d=c.length;e<d;e++)K(a,c[e]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;
-M(a);a._bInitComplete||ta(a,b);a.bAjaxDataGet=!0;C(a,!1)}function sa(a,b){var c=h.isPlainObject(a.ajax)&&a.ajax.dataSrc!==k?a.ajax.dataSrc:a.sAjaxDataProp;return"data"===c?b.aaData||b[c]:""!==c?R(c)(b):b}function ob(a){var b=a.oClasses,c=a.sTableId,e=a.oLanguage,d=a.oPreviousSearch,f=a.aanFeatures,g='<input type="search" class="'+b.sFilterInput+'"/>',j=e.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_",g):j+g,b=h("<div/>",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("<label/>").append(j)),
-f=function(){var b=!this.value?"":this.value;b!=d.sSearch&&(fa(a,{sSearch:b,bRegex:d.bRegex,bSmart:d.bSmart,bCaseInsensitive:d.bCaseInsensitive}),a._iDisplayStart=0,M(a))},g=null!==a.searchDelay?a.searchDelay:"ssp"===B(a)?400:0,i=h("input",b).val(d.sSearch).attr("placeholder",e.sSearchPlaceholder).bind("keyup.DT search.DT input.DT paste.DT cut.DT",g?ua(f,g):f).bind("keypress.DT",function(a){if(13==a.keyCode)return!1}).attr("aria-controls",c);h(a.nTable).on("search.dt.DT",function(b,c){if(a===c)try{i[0]!==
-Q.activeElement&&i.val(d.sSearch)}catch(f){}});return b[0]}function fa(a,b,c){var e=a.oPreviousSearch,d=a.aoPreSearchCols,f=function(a){e.sSearch=a.sSearch;e.bRegex=a.bRegex;e.bSmart=a.bSmart;e.bCaseInsensitive=a.bCaseInsensitive};Ha(a);if("ssp"!=B(a)){vb(a,b.sSearch,c,b.bEscapeRegex!==k?!b.bEscapeRegex:b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<d.length;b++)wb(a,d[b].sSearch,b,d[b].bEscapeRegex!==k?!d[b].bEscapeRegex:d[b].bRegex,d[b].bSmart,d[b].bCaseInsensitive);xb(a)}else f(b);a.bFiltered=
-!0;w(a,null,"search",[a])}function xb(a){for(var b=m.ext.search,c=a.aiDisplay,e,d,f=0,g=b.length;f<g;f++){for(var j=[],i=0,h=c.length;i<h;i++)d=c[i],e=a.aoData[d],b[f](a,e._aFilterData,d,e._aData,i)&&j.push(d);c.length=0;c.push.apply(c,j)}}function wb(a,b,c,e,d,f){if(""!==b)for(var g=a.aiDisplay,e=Qa(b,e,d,f),d=g.length-1;0<=d;d--)b=a.aoData[g[d]]._aFilterData[c],e.test(b)||g.splice(d,1)}function vb(a,b,c,e,d,f){var e=Qa(b,e,d,f),d=a.oPreviousSearch.sSearch,f=a.aiDisplayMaster,g;0!==m.ext.search.length&&
-(c=!0);g=yb(a);if(0>=b.length)a.aiDisplay=f.slice();else{if(g||c||d.length>b.length||0!==b.indexOf(d)||a.bSorted)a.aiDisplay=f.slice();b=a.aiDisplay;for(c=b.length-1;0<=c;c--)e.test(a.aoData[b[c]]._sFilterRow)||b.splice(c,1)}}function Qa(a,b,c,e){a=b?a:va(a);c&&(a="^(?=.*?"+h.map(a.match(/"[^"]+"|[^ ]+/g)||[""],function(a){if('"'===a.charAt(0))var b=a.match(/^"(.*)"$/),a=b?b[1]:a;return a.replace('"',"")}).join(")(?=.*?")+").*$");return RegExp(a,e?"i":"")}function va(a){return a.replace(Yb,"\\$1")}
-function yb(a){var b=a.aoColumns,c,e,d,f,g,j,i,h,l=m.ext.type.search;c=!1;e=0;for(f=a.aoData.length;e<f;e++)if(h=a.aoData[e],!h._aFilterData){j=[];d=0;for(g=b.length;d<g;d++)c=b[d],c.bSearchable?(i=x(a,e,d,"filter"),l[c.sType]&&(i=l[c.sType](i)),null===i&&(i=""),"string"!==typeof i&&i.toString&&(i=i.toString())):i="",i.indexOf&&-1!==i.indexOf("&")&&(wa.innerHTML=i,i=Zb?wa.textContent:wa.innerText),i.replace&&(i=i.replace(/[\r\n]/g,"")),j.push(i);h._aFilterData=j;h._sFilterRow=j.join(" ");c=!0}return c}
-function zb(a){return{search:a.sSearch,smart:a.bSmart,regex:a.bRegex,caseInsensitive:a.bCaseInsensitive}}function Ab(a){return{sSearch:a.search,bSmart:a.smart,bRegex:a.regex,bCaseInsensitive:a.caseInsensitive}}function rb(a){var b=a.sTableId,c=a.aanFeatures.i,e=h("<div/>",{"class":a.oClasses.sInfo,id:!c?b+"_info":null});c||(a.aoDrawCallback.push({fn:Bb,sName:"information"}),e.attr("role","status").attr("aria-live","polite"),h(a.nTable).attr("aria-describedby",b+"_info"));return e[0]}function Bb(a){var b=
-a.aanFeatures.i;if(0!==b.length){var c=a.oLanguage,e=a._iDisplayStart+1,d=a.fnDisplayEnd(),f=a.fnRecordsTotal(),g=a.fnRecordsDisplay(),j=g?c.sInfo:c.sInfoEmpty;g!==f&&(j+=" "+c.sInfoFiltered);j+=c.sInfoPostFix;j=Cb(a,j);c=c.fnInfoCallback;null!==c&&(j=c.call(a.oInstance,a,e,d,f,g,j));h(b).html(j)}}function Cb(a,b){var c=a.fnFormatNumber,e=a._iDisplayStart+1,d=a._iDisplayLength,f=a.fnRecordsDisplay(),g=-1===d;return b.replace(/_START_/g,c.call(a,e)).replace(/_END_/g,c.call(a,a.fnDisplayEnd())).replace(/_MAX_/g,
-c.call(a,a.fnRecordsTotal())).replace(/_TOTAL_/g,c.call(a,f)).replace(/_PAGE_/g,c.call(a,g?1:Math.ceil(e/d))).replace(/_PAGES_/g,c.call(a,g?1:Math.ceil(f/d)))}function ga(a){var b,c,e=a.iInitDisplayStart,d=a.aoColumns,f;c=a.oFeatures;if(a.bInitialised){mb(a);jb(a);ea(a,a.aoHeader);ea(a,a.aoFooter);C(a,!0);c.bAutoWidth&&Ga(a);b=0;for(c=d.length;b<c;b++)f=d[b],f.sWidth&&(f.nTh.style.width=s(f.sWidth));N(a);d=B(a);"ssp"!=d&&("ajax"==d?ra(a,[],function(c){var f=sa(a,c);for(b=0;b<f.length;b++)K(a,f[b]);
-a.iInitDisplayStart=e;N(a);C(a,!1);ta(a,c)},a):(C(a,!1),ta(a)))}else setTimeout(function(){ga(a)},200)}function ta(a,b){a._bInitComplete=!0;b&&X(a);w(a,"aoInitComplete","init",[a,b])}function Ra(a,b){var c=parseInt(b,10);a._iDisplayLength=c;Sa(a);w(a,null,"length",[a,c])}function nb(a){for(var b=a.oClasses,c=a.sTableId,e=a.aLengthMenu,d=h.isArray(e[0]),f=d?e[0]:e,e=d?e[1]:e,d=h("<select/>",{name:c+"_length","aria-controls":c,"class":b.sLengthSelect}),g=0,j=f.length;g<j;g++)d[0][g]=new Option(e[g],
-f[g]);var i=h("<div><label/></div>").addClass(b.sLength);a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",d[0].outerHTML));h("select",i).val(a._iDisplayLength).bind("change.DT",function(){Ra(a,h(this).val());M(a)});h(a.nTable).bind("length.dt.DT",function(b,c,f){a===c&&h("select",i).val(f)});return i[0]}function sb(a){var b=a.sPaginationType,c=m.ext.pager[b],e="function"===typeof c,d=function(a){M(a)},b=h("<div/>").addClass(a.oClasses.sPaging+b)[0],
-f=a.aanFeatures;e||c.fnInit(a,b,d);f.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(a){if(e){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),q,l=0;for(q=f.p.length;l<q;l++)Pa(a,"pageButton")(a,f.p[l],l,h,b,i)}else c.fnUpdate(a,d)},sName:"pagination"}));return b}function Ta(a,b,c){var e=a._iDisplayStart,d=a._iDisplayLength,f=a.fnRecordsDisplay();0===f||-1===d?e=0:"number"===typeof b?(e=b*d,e>f&&(e=0)):
-"first"==b?e=0:"previous"==b?(e=0<=d?e-d:0,0>e&&(e=0)):"next"==b?e+d<f&&(e+=d):"last"==b?e=Math.floor((f-1)/d)*d:I(a,0,"Unknown paging action: "+b,5);b=a._iDisplayStart!==e;a._iDisplayStart=e;b&&(w(a,null,"page",[a]),c&&M(a));return b}function pb(a){return h("<div/>",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]}function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");w(a,
-null,"processing",[a,b])}function qb(a){var b=h(a.nTable);b.attr("role","grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var e=c.sX,d=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),o=h(b[0].cloneNode(!1)),l=b.children("tfoot");c.sX&&"100%"===b.attr("width")&&b.removeAttr("width");l.length||(l=null);c=h("<div/>",{"class":f.sScrollWrapper}).append(h("<div/>",{"class":f.sScrollHead}).css({overflow:"hidden",position:"relative",border:0,
-width:e?!e?null:s(e):"100%"}).append(h("<div/>",{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("<div/>",{"class":f.sScrollBody}).css({overflow:"auto",height:!d?null:s(d),width:!e?null:s(e)}).append(b));l&&c.append(h("<div/>",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:e?!e?null:s(e):"100%"}).append(h("<div/>",{"class":f.sScrollFootInner}).append(o.removeAttr("id").css("margin-left",
-0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=c.children(),q=b[0],f=b[1],n=l?b[2]:null;if(e)h(f).on("scroll.DT",function(){var a=this.scrollLeft;q.scrollLeft=a;l&&(n.scrollLeft=a)});a.nScrollHead=q;a.nScrollBody=f;a.nScrollFoot=n;a.aoDrawCallback.push({fn:Y,sName:"scrolling"});return c[0]}function Y(a){var b=a.oScroll,c=b.sX,e=b.sXInner,d=b.sY,f=b.iBarWidth,g=h(a.nScrollHead),j=g[0].style,i=g.children("div"),o=i[0].style,l=i.children("table"),i=a.nScrollBody,q=h(i),n=i.style,
-k=h(a.nScrollFoot).children("div"),p=k.children("table"),m=h(a.nTHead),r=h(a.nTable),t=r[0],O=t.style,L=a.nTFoot?h(a.nTFoot):null,ha=a.oBrowser,w=ha.bScrollOversize,v,u,y,x,z,A=[],B=[],C=[],D,E=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};r.children("thead, tfoot").remove();z=m.clone().prependTo(r);v=m.find("tr");y=z.find("tr");z.find("th, td").removeAttr("tabindex");L&&(x=L.clone().prependTo(r),u=L.find("tr"),x=x.find("tr"));
-c||(n.width="100%",g[0].style.width="100%");h.each(qa(a,z),function(b,c){D=la(a,b);c.style.width=a.aoColumns[D].sWidth});L&&G(function(a){a.style.width=""},x);b.bCollapse&&""!==d&&(n.height=q[0].offsetHeight+m[0].offsetHeight+"px");g=r.outerWidth();if(""===c){if(O.width="100%",w&&(r.find("tbody").height()>i.offsetHeight||"scroll"==q.css("overflow-y")))O.width=s(r.outerWidth()-f)}else""!==e?O.width=s(e):g==q.width()&&q.height()<r.height()?(O.width=s(g-f),r.outerWidth()>g-f&&(O.width=s(g))):O.width=
-s(g);g=r.outerWidth();G(E,y);G(function(a){C.push(a.innerHTML);A.push(s(h(a).css("width")))},y);G(function(a,b){a.style.width=A[b]},v);h(y).height(0);L&&(G(E,x),G(function(a){B.push(s(h(a).css("width")))},x),G(function(a,b){a.style.width=B[b]},u),h(x).height(0));G(function(a,b){a.innerHTML='<div class="dataTables_sizing" style="height:0;overflow:hidden;">'+C[b]+"</div>";a.style.width=A[b]},y);L&&G(function(a,b){a.innerHTML="";a.style.width=B[b]},x);if(r.outerWidth()<g){u=i.scrollHeight>i.offsetHeight||
-"scroll"==q.css("overflow-y")?g+f:g;if(w&&(i.scrollHeight>i.offsetHeight||"scroll"==q.css("overflow-y")))O.width=s(u-f);(""===c||""!==e)&&I(a,1,"Possible column misalignment",6)}else u="100%";n.width=s(u);j.width=s(u);L&&(a.nScrollFoot.style.width=s(u));!d&&w&&(n.height=s(t.offsetHeight+f));d&&b.bCollapse&&(n.height=s(d),b=c&&t.offsetWidth>i.offsetWidth?f:0,t.offsetHeight<i.offsetHeight&&(n.height=s(t.offsetHeight+b)));b=r.outerWidth();l[0].style.width=s(b);o.width=s(b);l=r.height()>i.clientHeight||
-"scroll"==q.css("overflow-y");ha="padding"+(ha.bScrollbarLeft?"Left":"Right");o[ha]=l?f+"px":"0px";L&&(p[0].style.width=s(b),k[0].style.width=s(b),k[0].style[ha]=l?f+"px":"0px");q.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)i.scrollTop=0}function G(a,b,c){for(var e=0,d=0,f=b.length,g,j;d<f;){g=b[d].firstChild;for(j=c?c[d].firstChild:null;g;)1===g.nodeType&&(c?a(g,j,e):a(g,e),e++),g=g.nextSibling,j=c?j.nextSibling:null;d++}}function Ga(a){var b=a.nTable,c=a.aoColumns,e=a.oScroll,d=e.sY,f=e.sX,
-g=e.sXInner,j=c.length,e=Z(a,"bVisible"),i=h("th",a.nTHead),o=b.getAttribute("width"),l=b.parentNode,k=!1,n,m;(n=b.style.width)&&-1!==n.indexOf("%")&&(o=n);for(n=0;n<e.length;n++)m=c[e[n]],null!==m.sWidth&&(m.sWidth=Db(m.sWidthOrig,l),k=!0);if(!k&&!f&&!d&&j==aa(a)&&j==i.length)for(n=0;n<j;n++)c[n].sWidth=s(i.eq(n).width());else{j=h(b).clone().css("visibility","hidden").removeAttr("id");j.find("tbody tr").remove();var p=h("<tr/>").appendTo(j.find("tbody"));j.find("tfoot th, tfoot td").css("width",
-"");i=qa(a,j.find("thead")[0]);for(n=0;n<e.length;n++)m=c[e[n]],i[n].style.width=null!==m.sWidthOrig&&""!==m.sWidthOrig?s(m.sWidthOrig):"";if(a.aoData.length)for(n=0;n<e.length;n++)k=e[n],m=c[k],h(Eb(a,k)).clone(!1).append(m.sContentPadding).appendTo(p);j.appendTo(l);f&&g?j.width(g):f?(j.css("width","auto"),j.width()<l.offsetWidth&&j.width(l.offsetWidth)):d?j.width(l.offsetWidth):o&&j.width(o);Fb(a,j[0]);if(f){for(n=g=0;n<e.length;n++)m=c[e[n]],d=h(i[n]).outerWidth(),g+=null===m.sWidthOrig?d:parseInt(m.sWidth,
-10)+d-h(i[n]).width();j.width(s(g));b.style.width=s(g)}for(n=0;n<e.length;n++)if(m=c[e[n]],d=h(i[n]).width())m.sWidth=s(d);b.style.width=s(j.css("width"));j.remove()}o&&(b.style.width=s(o));if((o||f)&&!a._reszEvt)b=function(){h(Ea).bind("resize.DT-"+a.sInstance,ua(function(){X(a)}))},a.oBrowser.bScrollOversize?setTimeout(b,1E3):b(),a._reszEvt=!0}function ua(a,b){var c=b!==k?b:200,e,d;return function(){var b=this,g=+new Date,j=arguments;e&&g<e+c?(clearTimeout(d),d=setTimeout(function(){e=k;a.apply(b,
-j)},c)):(e=g,a.apply(b,j))}}function Db(a,b){if(!a)return 0;var c=h("<div/>").css("width",s(a)).appendTo(b||Q.body),e=c[0].offsetWidth;c.remove();return e}function Fb(a,b){var c=a.oScroll;if(c.sX||c.sY)c=!c.sX?c.iBarWidth:0,b.style.width=s(h(b).outerWidth()-c)}function Eb(a,b){var c=Gb(a,b);if(0>c)return null;var e=a.aoData[c];return!e.nTr?h("<td/>").html(x(a,c,b,"display"))[0]:e.anCells[b]}function Gb(a,b){for(var c,e=-1,d=-1,f=0,g=a.aoData.length;f<g;f++)c=x(a,f,b,"display")+"",c=c.replace($b,""),
-c.length>e&&(e=c.length,d=f);return d}function s(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function Hb(){var a=m.__scrollbarWidth;if(a===k){var b=h("<p/>").css({position:"absolute",top:0,left:0,width:"100%",height:150,padding:0,overflow:"scroll",visibility:"hidden"}).appendTo("body"),a=b[0].offsetWidth-b[0].clientWidth;m.__scrollbarWidth=a;b.remove()}return a}function U(a){var b,c,e=[],d=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var o=[];
-f=function(a){a.length&&!h.isArray(a[0])?o.push(a):o.push.apply(o,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;a<o.length;a++){i=o[a][0];f=d[i].aDataSort;b=0;for(c=f.length;b<c;b++)g=f[b],j=d[g].sType||"string",o[a]._idx===k&&(o[a]._idx=h.inArray(o[a][1],d[g].asSorting)),e.push({src:i,col:g,dir:o[a][1],index:o[a]._idx,type:j,formatter:m.ext.type.order[j+"-pre"]})}return e}function lb(a){var b,c,e=[],d=m.ext.type.order,f=a.aoData,g=0,j,i=a.aiDisplayMaster,h;
-Ha(a);h=U(a);b=0;for(c=h.length;b<c;b++)j=h[b],j.formatter&&g++,Ib(a,j.col);if("ssp"!=B(a)&&0!==h.length){b=0;for(c=i.length;b<c;b++)e[i[b]]=b;g===h.length?i.sort(function(a,b){var c,d,g,j,i=h.length,k=f[a]._aSortData,m=f[b]._aSortData;for(g=0;g<i;g++)if(j=h[g],c=k[j.col],d=m[j.col],c=c<d?-1:c>d?1:0,0!==c)return"asc"===j.dir?c:-c;c=e[a];d=e[b];return c<d?-1:c>d?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,m=f[a]._aSortData,r=f[b]._aSortData;for(j=0;j<k;j++)if(i=h[j],c=m[i.col],g=r[i.col],i=d[i.type+
-"-"+i.dir]||d["string-"+i.dir],c=i(c,g),0!==c)return c;c=e[a];g=e[b];return c<g?-1:c>g?1:0})}a.bSorted=!0}function Jb(a){for(var b,c,e=a.aoColumns,d=U(a),a=a.oLanguage.oAria,f=0,g=e.length;f<g;f++){c=e[f];var j=c.asSorting;b=c.sTitle.replace(/<.*?>/g,"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0<d.length&&d[0].col==f?(i.setAttribute("aria-sort","asc"==d[0].dir?"ascending":"descending"),c=j[d[0].index+1]||j[0]):c=j[0],b+="asc"===c?a.sSortAscending:a.sSortDescending);i.setAttribute("aria-label",
-b)}}function Ua(a,b,c,e){var d=a.aaSorting,f=a.aoColumns[b].asSorting,g=function(a,b){var c=a._idx;c===k&&(c=h.inArray(a[1],f));return c+1<f.length?c+1:b?null:0};"number"===typeof d[0]&&(d=a.aaSorting=[d]);c&&a.oFeatures.bSortMulti?(c=h.inArray(b,D(d,"0")),-1!==c?(b=g(d[c],!0),null===b&&1===d.length&&(b=0),null===b?d.splice(c,1):(d[c][1]=f[b],d[c]._idx=b)):(d.push([b,f[0],0]),d[d.length-1]._idx=0)):d.length&&d[0][0]==b?(b=g(d[0]),d.length=1,d[0][1]=f[b],d[0]._idx=b):(d.length=0,d.push([b,f[0]]),d[0]._idx=
-0);N(a);"function"==typeof e&&e(a)}function Oa(a,b,c,e){var d=a.aoColumns[c];Va(b,{},function(b){!1!==d.bSortable&&(a.oFeatures.bProcessing?(C(a,!0),setTimeout(function(){Ua(a,c,b.shiftKey,e);"ssp"!==B(a)&&C(a,!1)},0)):Ua(a,c,b.shiftKey,e))})}function xa(a){var b=a.aLastSort,c=a.oClasses.sSortColumn,e=U(a),d=a.oFeatures,f,g;if(d.bSort&&d.bSortClasses){d=0;for(f=b.length;d<f;d++)g=b[d].src,h(D(a.aoData,"anCells",g)).removeClass(c+(2>d?d+1:3));d=0;for(f=e.length;d<f;d++)g=e[d].src,h(D(a.aoData,"anCells",
-g)).addClass(c+(2>d?d+1:3))}a.aLastSort=e}function Ib(a,b){var c=a.aoColumns[b],e=m.ext.order[c.sSortDataType],d;e&&(d=e.call(a.oInstance,a,b,$(a,b)));for(var f,g=m.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j<i;j++)if(c=a.aoData[j],c._aSortData||(c._aSortData=[]),!c._aSortData[b]||e)f=e?d[j]:x(a,j,b,"sort"),c._aSortData[b]=g?g(f):f}function ya(a){if(a.oFeatures.bStateSave&&!a.bDestroying){var b={time:+new Date,start:a._iDisplayStart,length:a._iDisplayLength,order:h.extend(!0,[],a.aaSorting),
-search:zb(a.oPreviousSearch),columns:h.map(a.aoColumns,function(b,e){return{visible:b.bVisible,search:zb(a.aoPreSearchCols[e])}})};w(a,"aoStateSaveParams","stateSaveParams",[a,b]);a.oSavedState=b;a.fnStateSaveCallback.call(a.oInstance,a,b)}}function Kb(a){var b,c,e=a.aoColumns;if(a.oFeatures.bStateSave){var d=a.fnStateLoadCallback.call(a.oInstance,a);if(d&&d.time&&(b=w(a,"aoStateLoadParams","stateLoadParams",[a,d]),-1===h.inArray(!1,b)&&(b=a.iStateDuration,!(0<b&&d.time<+new Date-1E3*b)&&e.length===
-d.columns.length))){a.oLoadedState=h.extend(!0,{},d);d.start!==k&&(a._iDisplayStart=d.start,a.iInitDisplayStart=d.start);d.length!==k&&(a._iDisplayLength=d.length);d.order!==k&&(a.aaSorting=[],h.each(d.order,function(b,c){a.aaSorting.push(c[0]>=e.length?[0,c[1]]:c)}));d.search!==k&&h.extend(a.oPreviousSearch,Ab(d.search));b=0;for(c=d.columns.length;b<c;b++){var f=d.columns[b];f.visible!==k&&(e[b].bVisible=f.visible);f.search!==k&&h.extend(a.aoPreSearchCols[b],Ab(f.search))}w(a,"aoStateLoaded","stateLoaded",
-[a,d])}}}function za(a){var b=m.settings,a=h.inArray(a,D(b,"nTable"));return-1!==a?b[a]:null}function I(a,b,c,e){c="DataTables warning: "+(null!==a?"table id="+a.sTableId+" - ":"")+c;e&&(c+=". For more information about this error, please see http://datatables.net/tn/"+e);if(b)Ea.console&&console.log&&console.log(c);else if(b=m.ext,b=b.sErrMode||b.errMode,w(a,null,"error",[a,e,c]),"alert"==b)alert(c);else{if("throw"==b)throw Error(c);"function"==typeof b&&b(a,e,c)}}function E(a,b,c,e){h.isArray(c)?
-h.each(c,function(c,f){h.isArray(f)?E(a,b,f[0],f[1]):E(a,b,f)}):(e===k&&(e=c),b[c]!==k&&(a[e]=b[c]))}function Lb(a,b,c){var e,d;for(d in b)b.hasOwnProperty(d)&&(e=b[d],h.isPlainObject(e)?(h.isPlainObject(a[d])||(a[d]={}),h.extend(!0,a[d],e)):a[d]=c&&"data"!==d&&"aaData"!==d&&h.isArray(e)?e.slice():e);return a}function Va(a,b,c){h(a).bind("click.DT",b,function(b){a.blur();c(b)}).bind("keypress.DT",b,function(a){13===a.which&&(a.preventDefault(),c(a))}).bind("selectstart.DT",function(){return!1})}function z(a,
-b,c,e){c&&a[b].push({fn:c,sName:e})}function w(a,b,c,e){var d=[];b&&(d=h.map(a[b].slice().reverse(),function(b){return b.fn.apply(a.oInstance,e)}));null!==c&&(b=h.Event(c+".dt"),h(a.nTable).trigger(b,e),d.push(b.result));return d}function Sa(a){var b=a._iDisplayStart,c=a.fnDisplayEnd(),e=a._iDisplayLength;b>=c&&(b=c-e);b-=b%e;if(-1===e||0>b)b=0;a._iDisplayStart=b}function Pa(a,b){var c=a.renderer,e=m.ext.renderer[b];return h.isPlainObject(c)&&c[b]?e[c[b]]||e._:"string"===typeof c?e[c]||e._:e._}function B(a){return a.oFeatures.bServerSide?
-"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function Wa(a,b){var c=[],c=Mb.numbers_length,e=Math.floor(c/2);b<=c?c=V(0,b):a<=e?(c=V(0,c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-e?c=V(b-(c-2),b):(c=V(a-e+2,a+e-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function db(a){h.each({num:function(b){return Aa(b,a)},"num-fmt":function(b){return Aa(b,a,Xa)},"html-num":function(b){return Aa(b,a,Ba)},"html-num-fmt":function(b){return Aa(b,a,Ba,Xa)}},function(b,
-c){u.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(u.type.search[b+a]=u.type.search.html)})}function Nb(a){return function(){var b=[za(this[m.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return m.ext.internal[a].apply(this,b)}}var m,u,t,r,v,Ya={},Ob=/[\r\n]/g,Ba=/<.*?>/g,ac=/^[\w\+\-]/,bc=/[\w\+\-]$/,Yb=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),Xa=/[',$\u00a3\u20ac\u00a5%\u2009\u202F\u20BD\u20a9\u20BArfk]/gi,J=function(a){return!a||!0===a||
-"-"===a?!0:!1},Pb=function(a){var b=parseInt(a,10);return!isNaN(b)&&isFinite(a)?b:null},Qb=function(a,b){Ya[b]||(Ya[b]=RegExp(va(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace(Ya[b],"."):a},Za=function(a,b,c){var e="string"===typeof a;if(J(a))return!0;b&&e&&(a=Qb(a,b));c&&e&&(a=a.replace(Xa,""));return!isNaN(parseFloat(a))&&isFinite(a)},Rb=function(a,b,c){return J(a)?!0:!(J(a)||"string"===typeof a)?null:Za(a.replace(Ba,""),b,c)?!0:null},D=function(a,b,c){var e=[],d=0,f=a.length;
-if(c!==k)for(;d<f;d++)a[d]&&a[d][b]&&e.push(a[d][b][c]);else for(;d<f;d++)a[d]&&e.push(a[d][b]);return e},ia=function(a,b,c,e){var d=[],f=0,g=b.length;if(e!==k)for(;f<g;f++)a[b[f]][c]&&d.push(a[b[f]][c][e]);else for(;f<g;f++)d.push(a[b[f]][c]);return d},V=function(a,b){var c=[],e;b===k?(b=0,e=a):(e=b,b=a);for(var d=b;d<e;d++)c.push(d);return c},Sb=function(a){for(var b=[],c=0,e=a.length;c<e;c++)a[c]&&b.push(a[c]);return b},Na=function(a){var b=[],c,e,d=a.length,f,g=0;e=0;a:for(;e<d;e++){c=a[e];for(f=
-0;f<g;f++)if(b[f]===c)continue a;b.push(c);g++}return b},A=function(a,b,c){a[b]!==k&&(a[c]=a[b])},ba=/\[.*?\]$/,T=/\(\)$/,wa=h("<div>")[0],Zb=wa.textContent!==k,$b=/<.*?>/g;m=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new t(za(this[u.iApiIndex])):new t(this)};this.fnAddData=function(a,b){var c=this.api(!0),e=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b===
-k||b)&&c.draw();return e.flatten().toArray()};this.fnAdjustColumnSizing=function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],e=c.oScroll;a===k||a?b.draw(!1):(""!==e.sX||""!==e.sY)&&Y(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a,b,c){var e=this.api(!0),a=e.rows(a),d=a.settings()[0],h=d.aoData[a[0][0]];a.remove();b&&b.call(this,d,h);(c===k||c)&&e.draw();return h};
-this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};this.fnFilter=function(a,b,c,e,d,h){d=this.api(!0);null===b||b===k?d.search(a,c,e,h):d.column(b).search(a,c,e,h);d.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var e=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==e||"th"==e?c.cell(a,b).data():c.row(a).data()||null}return c.data().toArray()};this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()};
-this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]};this.fnPageChange=function(a,b){var c=this.api(!0).page(a);(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c===
-k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return za(this[u.iApiIndex])};this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,e,d){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(d===k||d)&&h.columns.adjust();(e===k||e)&&h.draw();return 0};this.fnVersionCheck=u.fnVersionCheck;var b=this,c=a===k,e=this.length;c&&(a={});this.oApi=this.internal=u.internal;for(var d in m.ext.internal)d&&
-(this[d]=Nb(d));this.each(function(){var d={},d=1<e?Lb(d,a,!0):a,g=0,j,i=this.getAttribute("id"),o=!1,l=m.defaults,q=h(this);if("table"!=this.nodeName.toLowerCase())I(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{eb(l);fb(l.column);H(l,l,!0);H(l.column,l.column,!0);H(l,h.extend(d,q.data()));var n=m.settings,g=0;for(j=n.length;g<j;g++){var r=n[g];if(r.nTable==this||r.nTHead.parentNode==this||r.nTFoot&&r.nTFoot.parentNode==this){g=d.bRetrieve!==k?d.bRetrieve:l.bRetrieve;if(c||g)return r.oInstance;
-if(d.bDestroy!==k?d.bDestroy:l.bDestroy){r.oInstance.fnDestroy();break}else{I(r,0,"Cannot reinitialise DataTable",3);return}}if(r.sTableId==this.id){n.splice(g,1);break}}if(null===i||""===i)this.id=i="DataTables_Table_"+m.ext._unique++;var p=h.extend(!0,{},m.models.oSettings,{sDestroyWidth:q[0].style.width,sInstance:i,sTableId:i});p.nTable=this;p.oApi=b.internal;p.oInit=d;n.push(p);p.oInstance=1===b.length?b:q.dataTable();eb(d);d.oLanguage&&P(d.oLanguage);d.aLengthMenu&&!d.iDisplayLength&&(d.iDisplayLength=
-h.isArray(d.aLengthMenu[0])?d.aLengthMenu[0][0]:d.aLengthMenu[0]);d=Lb(h.extend(!0,{},l),d);E(p.oFeatures,d,"bPaginate bLengthChange bFilter bSort bSortMulti bInfo bProcessing bAutoWidth bSortClasses bServerSide bDeferRender".split(" "));E(p,d,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod","aaSorting","aaSortingFixed","aLengthMenu","sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback",
-"renderer","searchDelay",["iCookieDuration","iStateDuration"],["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"],["bJQueryUI","bJUI"]]);E(p.oScroll,d,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]);E(p.oLanguage,d,"fnInfoCallback");z(p,"aoDrawCallback",d.fnDrawCallback,"user");z(p,"aoServerParams",d.fnServerParams,"user");z(p,"aoStateSaveParams",d.fnStateSaveParams,"user");z(p,"aoStateLoadParams",
-d.fnStateLoadParams,"user");z(p,"aoStateLoaded",d.fnStateLoaded,"user");z(p,"aoRowCallback",d.fnRowCallback,"user");z(p,"aoRowCreatedCallback",d.fnCreatedRow,"user");z(p,"aoHeaderCallback",d.fnHeaderCallback,"user");z(p,"aoFooterCallback",d.fnFooterCallback,"user");z(p,"aoInitComplete",d.fnInitComplete,"user");z(p,"aoPreDrawCallback",d.fnPreDrawCallback,"user");i=p.oClasses;d.bJQueryUI?(h.extend(i,m.ext.oJUIClasses,d.oClasses),d.sDom===l.sDom&&"lfrtip"===l.sDom&&(p.sDom='<"H"lfr>t<"F"ip>'),p.renderer)?
-h.isPlainObject(p.renderer)&&!p.renderer.header&&(p.renderer.header="jqueryui"):p.renderer="jqueryui":h.extend(i,m.ext.classes,d.oClasses);q.addClass(i.sTable);if(""!==p.oScroll.sX||""!==p.oScroll.sY)p.oScroll.iBarWidth=Hb();!0===p.oScroll.sX&&(p.oScroll.sX="100%");p.iInitDisplayStart===k&&(p.iInitDisplayStart=d.iDisplayStart,p._iDisplayStart=d.iDisplayStart);null!==d.iDeferLoading&&(p.bDeferLoading=!0,g=h.isArray(d.iDeferLoading),p._iRecordsDisplay=g?d.iDeferLoading[0]:d.iDeferLoading,p._iRecordsTotal=
-g?d.iDeferLoading[1]:d.iDeferLoading);var t=p.oLanguage;h.extend(!0,t,d.oLanguage);""!==t.sUrl&&(h.ajax({dataType:"json",url:t.sUrl,success:function(a){P(a);H(l.oLanguage,a);h.extend(true,t,a);ga(p)},error:function(){ga(p)}}),o=!0);null===d.asStripeClasses&&(p.asStripeClasses=[i.sStripeOdd,i.sStripeEven]);var g=p.asStripeClasses,s=q.children("tbody").find("tr").eq(0);-1!==h.inArray(!0,h.map(g,function(a){return s.hasClass(a)}))&&(h("tbody tr",this).removeClass(g.join(" ")),p.asDestroyStripes=g.slice());
-n=[];g=this.getElementsByTagName("thead");0!==g.length&&(da(p.aoHeader,g[0]),n=qa(p));if(null===d.aoColumns){r=[];g=0;for(j=n.length;g<j;g++)r.push(null)}else r=d.aoColumns;g=0;for(j=r.length;g<j;g++)Fa(p,n?n[g]:null);ib(p,d.aoColumnDefs,r,function(a,b){ka(p,a,b)});if(s.length){var u=function(a,b){return a.getAttribute("data-"+b)!==null?b:null};h.each(na(p,s[0]).cells,function(a,b){var c=p.aoColumns[a];if(c.mData===a){var d=u(b,"sort")||u(b,"order"),e=u(b,"filter")||u(b,"search");if(d!==null||e!==
-null){c.mData={_:a+".display",sort:d!==null?a+".@data-"+d:k,type:d!==null?a+".@data-"+d:k,filter:e!==null?a+".@data-"+e:k};ka(p,a)}}})}var v=p.oFeatures;d.bStateSave&&(v.bStateSave=!0,Kb(p,d),z(p,"aoDrawCallback",ya,"state_save"));if(d.aaSorting===k){n=p.aaSorting;g=0;for(j=n.length;g<j;g++)n[g][1]=p.aoColumns[g].asSorting[0]}xa(p);v.bSort&&z(p,"aoDrawCallback",function(){if(p.bSorted){var a=U(p),b={};h.each(a,function(a,c){b[c.src]=c.dir});w(p,null,"order",[p,a,b]);Jb(p)}});z(p,"aoDrawCallback",
-function(){(p.bSorted||B(p)==="ssp"||v.bDeferRender)&&xa(p)},"sc");gb(p);g=q.children("caption").each(function(){this._captionSide=q.css("caption-side")});j=q.children("thead");0===j.length&&(j=h("<thead/>").appendTo(this));p.nTHead=j[0];j=q.children("tbody");0===j.length&&(j=h("<tbody/>").appendTo(this));p.nTBody=j[0];j=q.children("tfoot");if(0===j.length&&0<g.length&&(""!==p.oScroll.sX||""!==p.oScroll.sY))j=h("<tfoot/>").appendTo(this);0===j.length||0===j.children().length?q.addClass(i.sNoFooter):
-0<j.length&&(p.nTFoot=j[0],da(p.aoFooter,p.nTFoot));if(d.aaData)for(g=0;g<d.aaData.length;g++)K(p,d.aaData[g]);else(p.bDeferLoading||"dom"==B(p))&&ma(p,h(p.nTBody).children("tr"));p.aiDisplay=p.aiDisplayMaster.slice();p.bInitialised=!0;!1===o&&ga(p)}});b=null;return this};var Tb=[],y=Array.prototype,cc=function(a){var b,c,e=m.settings,d=h.map(e,function(a){return a.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase())return b=h.inArray(a,d),-1!==b?[e[b]]:
-null;if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?c=h(a):a instanceof h&&(c=a)}else return[];if(c)return c.map(function(){b=h.inArray(this,d);return-1!==b?e[b]:null}).toArray()};t=function(a,b){if(!(this instanceof t))return new t(a,b);var c=[],e=function(a){(a=cc(a))&&c.push.apply(c,a)};if(h.isArray(a))for(var d=0,f=a.length;d<f;d++)e(a[d]);else e(a);this.context=Na(c);b&&this.push.apply(this,b.toArray?b.toArray():b);this.selector={rows:null,cols:null,opts:null};
-t.extend(this,this,Tb)};m.Api=t;t.prototype={any:function(){return 0!==this.flatten().length},concat:y.concat,context:[],each:function(a){for(var b=0,c=this.length;b<c;b++)a.call(this,this[b],b,this);return this},eq:function(a){var b=this.context;return b.length>a?new t(b[a],this[a]):null},filter:function(a){var b=[];if(y.filter)b=y.filter.call(this,a,this);else for(var c=0,e=this.length;c<e;c++)a.call(this,this[c],c,this)&&b.push(this[c]);return new t(this.context,b)},flatten:function(){var a=[];
-return new t(this.context,a.concat.apply(a,this.toArray()))},join:y.join,indexOf:y.indexOf||function(a,b){for(var c=b||0,e=this.length;c<e;c++)if(this[c]===a)return c;return-1},iterator:function(a,b,c,e){var d=[],f,g,h,i,o,l=this.context,q,n,m=this.selector;"string"===typeof a&&(e=c,c=b,b=a,a=!1);g=0;for(h=l.length;g<h;g++){var p=new t(l[g]);if("table"===b)f=c.call(p,l[g],g),f!==k&&d.push(f);else if("columns"===b||"rows"===b)f=c.call(p,l[g],this[g],g),f!==k&&d.push(f);else if("column"===b||"column-rows"===
-b||"row"===b||"cell"===b){n=this[g];"column-rows"===b&&(q=Ca(l[g],m.opts));i=0;for(o=n.length;i<o;i++)f=n[i],f="cell"===b?c.call(p,l[g],f.row,f.column,g,i):c.call(p,l[g],f,g,i,q),f!==k&&d.push(f)}}return d.length||e?(a=new t(l,a?d.concat.apply([],d):d),b=a.selector,b.rows=m.rows,b.cols=m.cols,b.opts=m.opts,a):this},lastIndexOf:y.lastIndexOf||function(a,b){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,map:function(a){var b=[];if(y.map)b=y.map.call(this,a,this);else for(var c=
-0,e=this.length;c<e;c++)b.push(a.call(this,this[c],c));return new t(this.context,b)},pluck:function(a){return this.map(function(b){return b[a]})},pop:y.pop,push:y.push,reduce:y.reduce||function(a,b){return hb(this,a,b,0,this.length,1)},reduceRight:y.reduceRight||function(a,b){return hb(this,a,b,this.length-1,-1,-1)},reverse:y.reverse,selector:null,shift:y.shift,sort:y.sort,splice:y.splice,toArray:function(){return y.slice.call(this)},to$:function(){return h(this)},toJQuery:function(){return h(this)},
-unique:function(){return new t(this.context,Na(this))},unshift:y.unshift};t.extend=function(a,b,c){if(c.length&&b&&(b instanceof t||b.__dt_wrapper)){var e,d,f,g=function(a,b,c){return function(){var d=b.apply(a,arguments);t.extend(d,d,c.methodExt);return d}};e=0;for(d=c.length;e<d;e++)f=c[e],b[f.name]="function"===typeof f.val?g(a,f.val,f):h.isPlainObject(f.val)?{}:f.val,b[f.name].__dt_wrapper=!0,t.extend(a,b[f.name],f.propExt)}};t.register=r=function(a,b){if(h.isArray(a))for(var c=0,e=a.length;c<
-e;c++)t.register(a[c],b);else for(var d=a.split("."),f=Tb,g,j,c=0,e=d.length;c<e;c++){g=(j=-1!==d[c].indexOf("()"))?d[c].replace("()",""):d[c];var i;a:{i=0;for(var o=f.length;i<o;i++)if(f[i].name===g){i=f[i];break a}i=null}i||(i={name:g,val:{},methodExt:[],propExt:[]},f.push(i));c===e-1?i.val=b:f=j?i.methodExt:i.propExt}};t.registerPlural=v=function(a,b,c){t.register(a,c);t.register(b,function(){var a=c.apply(this,arguments);return a===this?this:a instanceof t?a.length?h.isArray(a[0])?new t(a.context,
-a[0]):a[0]:k:a})};r("tables()",function(a){var b;if(a){b=t;var c=this.context;if("number"===typeof a)a=[c[a]];else var e=h.map(c,function(a){return a.nTable}),a=h(e).filter(a).map(function(){var a=h.inArray(this,e);return c[a]}).toArray();b=new b(a)}else b=this;return b});r("table()",function(a){var a=this.tables(a),b=a.context;return b.length?new t(b[0]):a});v("tables().nodes()","table().node()",function(){return this.iterator("table",function(a){return a.nTable},1)});v("tables().body()","table().body()",
-function(){return this.iterator("table",function(a){return a.nTBody},1)});v("tables().header()","table().header()",function(){return this.iterator("table",function(a){return a.nTHead},1)});v("tables().footer()","table().footer()",function(){return this.iterator("table",function(a){return a.nTFoot},1)});v("tables().containers()","table().container()",function(){return this.iterator("table",function(a){return a.nTableWrapper},1)});r("draw()",function(a){return this.iterator("table",function(b){N(b,
-!1===a)})});r("page()",function(a){return a===k?this.page.info().page:this.iterator("table",function(b){Ta(b,a)})});r("page.info()",function(){if(0===this.context.length)return k;var a=this.context[0],b=a._iDisplayStart,c=a._iDisplayLength,e=a.fnRecordsDisplay(),d=-1===c;return{page:d?0:Math.floor(b/c),pages:d?1:Math.ceil(e/c),start:b,end:a.fnDisplayEnd(),length:c,recordsTotal:a.fnRecordsTotal(),recordsDisplay:e}});r("page.len()",function(a){return a===k?0!==this.context.length?this.context[0]._iDisplayLength:
-k:this.iterator("table",function(b){Ra(b,a)})});var Ub=function(a,b,c){if(c){var e=new t(a);e.one("draw",function(){c(e.ajax.json())})}"ssp"==B(a)?N(a,b):(C(a,!0),ra(a,[],function(c){oa(a);for(var c=sa(a,c),e=0,g=c.length;e<g;e++)K(a,c[e]);N(a,b);C(a,!1)}))};r("ajax.json()",function(){var a=this.context;if(0<a.length)return a[0].json});r("ajax.params()",function(){var a=this.context;if(0<a.length)return a[0].oAjaxData});r("ajax.reload()",function(a,b){return this.iterator("table",function(c){Ub(c,
-!1===b,a)})});r("ajax.url()",function(a){var b=this.context;if(a===k){if(0===b.length)return k;b=b[0];return b.ajax?h.isPlainObject(b.ajax)?b.ajax.url:b.ajax:b.sAjaxSource}return this.iterator("table",function(b){h.isPlainObject(b.ajax)?b.ajax.url=a:b.ajax=a})});r("ajax.url().load()",function(a,b){return this.iterator("table",function(c){Ub(c,!1===b,a)})});var $a=function(a,b,c,e,d){var f=[],g,j,i,o,l,q;i=typeof b;if(!b||"string"===i||"function"===i||b.length===k)b=[b];i=0;for(o=b.length;i<o;i++){j=
-b[i]&&b[i].split?b[i].split(","):[b[i]];l=0;for(q=j.length;l<q;l++)(g=c("string"===typeof j[l]?h.trim(j[l]):j[l]))&&g.length&&f.push.apply(f,g)}a=u.selector[a];if(a.length){i=0;for(o=a.length;i<o;i++)f=a[i](e,d,f)}return f},ab=function(a){a||(a={});a.filter&&a.search===k&&(a.search=a.filter);return h.extend({search:"none",order:"current",page:"all"},a)},bb=function(a){for(var b=0,c=a.length;b<c;b++)if(0<a[b].length)return a[0]=a[b],a[0].length=1,a.length=1,a.context=[a.context[b]],a;a.length=0;return a},
-Ca=function(a,b){var c,e,d,f=[],g=a.aiDisplay;c=a.aiDisplayMaster;var j=b.search;e=b.order;d=b.page;if("ssp"==B(a))return"removed"===j?[]:V(0,c.length);if("current"==d){c=a._iDisplayStart;for(e=a.fnDisplayEnd();c<e;c++)f.push(g[c])}else if("current"==e||"applied"==e)f="none"==j?c.slice():"applied"==j?g.slice():h.map(c,function(a){return-1===h.inArray(a,g)?a:null});else if("index"==e||"original"==e){c=0;for(e=a.aoData.length;c<e;c++)"none"==j?f.push(c):(d=h.inArray(c,g),(-1===d&&"removed"==j||0<=d&&
-"applied"==j)&&f.push(c))}return f};r("rows()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=ab(b),c=this.iterator("table",function(c){var d=b;return $a("row",a,function(a){var b=Pb(a);if(b!==null&&!d)return[b];var j=Ca(c,d);if(b!==null&&h.inArray(b,j)!==-1)return[b];if(!a)return j;if(typeof a==="function")return h.map(j,function(b){var d=c.aoData[b];return a(b,d._aData,d.nTr)?b:null});b=Sb(ia(c.aoData,j,"nTr"));return a.nodeName&&h.inArray(a,b)!==-1?[a._DT_RowIndex]:h(b).filter(a).map(function(){return this._DT_RowIndex}).toArray()},
-c,d)},1);c.selector.rows=a;c.selector.opts=b;return c});r("rows().nodes()",function(){return this.iterator("row",function(a,b){return a.aoData[b].nTr||k},1)});r("rows().data()",function(){return this.iterator(!0,"rows",function(a,b){return ia(a.aoData,b,"_aData")},1)});v("rows().cache()","row().cache()",function(a){return this.iterator("row",function(b,c){var e=b.aoData[c];return"search"===a?e._aFilterData:e._aSortData},1)});v("rows().invalidate()","row().invalidate()",function(a){return this.iterator("row",
-function(b,c){ca(b,c,a)})});v("rows().indexes()","row().index()",function(){return this.iterator("row",function(a,b){return b},1)});v("rows().remove()","row().remove()",function(){var a=this;return this.iterator("row",function(b,c,e){var d=b.aoData;d.splice(c,1);for(var f=0,g=d.length;f<g;f++)null!==d[f].nTr&&(d[f].nTr._DT_RowIndex=f);h.inArray(c,b.aiDisplay);pa(b.aiDisplayMaster,c);pa(b.aiDisplay,c);pa(a[e],c,!1);Sa(b)})});r("rows.add()",function(a){var b=this.iterator("table",function(b){var c,
-f,g,h=[];f=0;for(g=a.length;f<g;f++)c=a[f],c.nodeName&&"TR"===c.nodeName.toUpperCase()?h.push(ma(b,c)[0]):h.push(K(b,c));return h},1),c=this.rows(-1);c.pop();c.push.apply(c,b.toArray());return c});r("row()",function(a,b){return bb(this.rows(a,b))});r("row().data()",function(a){var b=this.context;if(a===k)return b.length&&this.length?b[0].aoData[this[0]]._aData:k;b[0].aoData[this[0]]._aData=a;ca(b[0],this[0],"data");return this});r("row().node()",function(){var a=this.context;return a.length&&this.length?
-a[0].aoData[this[0]].nTr||null:null});r("row.add()",function(a){a instanceof h&&a.length&&(a=a[0]);var b=this.iterator("table",function(b){return a.nodeName&&"TR"===a.nodeName.toUpperCase()?ma(b,a)[0]:K(b,a)});return this.row(b[0])});var cb=function(a,b){var c=a.context;c.length&&(c=c[0].aoData[b!==k?b:a[0]],c._details&&(c._details.remove(),c._detailsShow=k,c._details=k))},Vb=function(a,b){var c=a.context;if(c.length&&a.length){var e=c[0].aoData[a[0]];if(e._details){(e._detailsShow=b)?e._details.insertAfter(e.nTr):
-e._details.detach();var d=c[0],f=new t(d),g=d.aoData;f.off("draw.dt.DT_details column-visibility.dt.DT_details destroy.dt.DT_details");0<D(g,"_details").length&&(f.on("draw.dt.DT_details",function(a,b){d===b&&f.rows({page:"current"}).eq(0).each(function(a){a=g[a];a._detailsShow&&a._details.insertAfter(a.nTr)})}),f.on("column-visibility.dt.DT_details",function(a,b){if(d===b)for(var c,e=aa(b),f=0,h=g.length;f<h;f++)c=g[f],c._details&&c._details.children("td[colspan]").attr("colspan",e)}),f.on("destroy.dt.DT_details",
-function(a,b){if(d===b)for(var c=0,e=g.length;c<e;c++)g[c]._details&&cb(f,c)}))}}};r("row().child()",function(a,b){var c=this.context;if(a===k)return c.length&&this.length?c[0].aoData[this[0]]._details:k;if(!0===a)this.child.show();else if(!1===a)cb(this);else if(c.length&&this.length){var e=c[0],c=c[0].aoData[this[0]],d=[],f=function(a,b){if(h.isArray(a)||a instanceof h)for(var c=0,k=a.length;c<k;c++)f(a[c],b);else a.nodeName&&"tr"===a.nodeName.toLowerCase()?d.push(a):(c=h("<tr><td/></tr>").addClass(b),
-h("td",c).addClass(b).html(a)[0].colSpan=aa(e),d.push(c[0]))};f(a,b);c._details&&c._details.remove();c._details=h(d);c._detailsShow&&c._details.insertAfter(c.nTr)}return this});r(["row().child.show()","row().child().show()"],function(){Vb(this,!0);return this});r(["row().child.hide()","row().child().hide()"],function(){Vb(this,!1);return this});r(["row().child.remove()","row().child().remove()"],function(){cb(this);return this});r("row().child.isShown()",function(){var a=this.context;return a.length&&
-this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var dc=/^(.+):(name|visIdx|visible)$/,Wb=function(a,b,c,e,d){for(var c=[],e=0,f=d.length;e<f;e++)c.push(x(a,d[e],b));return c};r("columns()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=ab(b),c=this.iterator("table",function(c){var d=a,f=b,g=c.aoColumns,j=D(g,"sName"),i=D(g,"nTh");return $a("column",d,function(a){var b=Pb(a);if(a==="")return V(g.length);if(b!==null)return[b>=0?b:g.length+b];if(typeof a==="function"){var d=Ca(c,
-f);return h.map(g,function(b,f){return a(f,Wb(c,f,0,0,d),i[f])?f:null})}var k=typeof a==="string"?a.match(dc):"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var m=h.map(g,function(a,b){return a.bVisible?b:null});return[m[m.length+b]]}return[la(c,b)];case "name":return h.map(j,function(a,b){return a===k[1]?b:null})}else return h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray()},c,f)},1);c.selector.cols=a;c.selector.opts=b;return c});v("columns().header()",
-"column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});v("columns().footer()","column().footer()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});v("columns().data()","column().data()",function(){return this.iterator("column-rows",Wb,1)});v("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData},1)});v("columns().cache()","column().cache()",
-function(a){return this.iterator("column-rows",function(b,c,e,d,f){return ia(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});v("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,e,d){return ia(a.aoData,d,"anCells",b)},1)});v("columns().visible()","column().visible()",function(a,b){return this.iterator("column",function(c,e){if(a===k)return c.aoColumns[e].bVisible;var d=c.aoColumns,f=d[e],g=c.aoData,j,i,m;if(a!==k&&f.bVisible!==a){if(a){var l=
-h.inArray(!0,D(d,"bVisible"),e+1);j=0;for(i=g.length;j<i;j++)m=g[j].nTr,d=g[j].anCells,m&&m.insertBefore(d[e],d[l]||null)}else h(D(c.aoData,"anCells",e)).detach();f.bVisible=a;ea(c,c.aoHeader);ea(c,c.aoFooter);if(b===k||b)X(c),(c.oScroll.sX||c.oScroll.sY)&&Y(c);w(c,null,"column-visibility",[c,e,a]);ya(c)}})});v("columns().indexes()","column().index()",function(a){return this.iterator("column",function(b,c){return"visible"===a?$(b,c):c},1)});r("columns.adjust()",function(){return this.iterator("table",
-function(a){X(a)},1)});r("column.index()",function(a,b){if(0!==this.context.length){var c=this.context[0];if("fromVisible"===a||"toData"===a)return la(c,b);if("fromData"===a||"toVisible"===a)return $(c,b)}});r("column()",function(a,b){return bb(this.columns(a,b))});r("cells()",function(a,b,c){h.isPlainObject(a)&&(a.row===k?(c=a,a=null):(c=b,b=null));h.isPlainObject(b)&&(c=b,b=null);if(null===b||b===k)return this.iterator("table",function(b){var d=a,e=ab(c),f=b.aoData,g=Ca(b,e),i=Sb(ia(f,g,"anCells")),
-j=h([].concat.apply([],i)),l,m=b.aoColumns.length,o,r,t,s,u,v;return $a("cell",d,function(a){var c=typeof a==="function";if(a===null||a===k||c){o=[];r=0;for(t=g.length;r<t;r++){l=g[r];for(s=0;s<m;s++){u={row:l,column:s};if(c){v=b.aoData[l];a(u,x(b,l,s),v.anCells?v.anCells[s]:null)&&o.push(u)}else o.push(u)}}return o}return h.isPlainObject(a)?[a]:j.filter(a).map(function(a,b){l=b.parentNode._DT_RowIndex;return{row:l,column:h.inArray(b,f[l].anCells)}}).toArray()},b,e)});var e=this.columns(b,c),d=this.rows(a,
-c),f,g,j,i,m,l=this.iterator("table",function(a,b){f=[];g=0;for(j=d[b].length;g<j;g++){i=0;for(m=e[b].length;i<m;i++)f.push({row:d[b][g],column:e[b][i]})}return f},1);h.extend(l.selector,{cols:b,rows:a,opts:c});return l});v("cells().nodes()","cell().node()",function(){return this.iterator("cell",function(a,b,c){return(a=a.aoData[b].anCells)?a[c]:k},1)});r("cells().data()",function(){return this.iterator("cell",function(a,b,c){return x(a,b,c)},1)});v("cells().cache()","cell().cache()",function(a){a=
-"search"===a?"_aFilterData":"_aSortData";return this.iterator("cell",function(b,c,e){return b.aoData[c][a][e]},1)});v("cells().render()","cell().render()",function(a){return this.iterator("cell",function(b,c,e){return x(b,c,e,a)},1)});v("cells().indexes()","cell().index()",function(){return this.iterator("cell",function(a,b,c){return{row:b,column:c,columnVisible:$(a,c)}},1)});v("cells().invalidate()","cell().invalidate()",function(a){return this.iterator("cell",function(b,c,e){ca(b,c,a,e)})});r("cell()",
-function(a,b,c){return bb(this.cells(a,b,c))});r("cell().data()",function(a){var b=this.context,c=this[0];if(a===k)return b.length&&c.length?x(b[0],c[0].row,c[0].column):k;Ia(b[0],c[0].row,c[0].column,a);ca(b[0],c[0].row,"data",c[0].column);return this});r("order()",function(a,b){var c=this.context;if(a===k)return 0!==c.length?c[0].aaSorting:k;"number"===typeof a?a=[[a,b]]:h.isArray(a[0])||(a=Array.prototype.slice.call(arguments));return this.iterator("table",function(b){b.aaSorting=a.slice()})});
-r("order.listener()",function(a,b,c){return this.iterator("table",function(e){Oa(e,a,b,c)})});r(["columns().order()","column().order()"],function(a){var b=this;return this.iterator("table",function(c,e){var d=[];h.each(b[e],function(b,c){d.push([c,a])});c.aaSorting=d})});r("search()",function(a,b,c,e){var d=this.context;return a===k?0!==d.length?d[0].oPreviousSearch.sSearch:k:this.iterator("table",function(d){d.oFeatures.bFilter&&fa(d,h.extend({},d.oPreviousSearch,{sSearch:a+"",bRegex:null===b?!1:
-b,bSmart:null===c?!0:c,bCaseInsensitive:null===e?!0:e}),1)})});v("columns().search()","column().search()",function(a,b,c,e){return this.iterator("column",function(d,f){var g=d.aoPreSearchCols;if(a===k)return g[f].sSearch;d.oFeatures.bFilter&&(h.extend(g[f],{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===e?!0:e}),fa(d,d.oPreviousSearch,1))})});r("state()",function(){return this.context.length?this.context[0].oSavedState:null});r("state.clear()",function(){return this.iterator("table",
-function(a){a.fnStateSaveCallback.call(a.oInstance,a,{})})});r("state.loaded()",function(){return this.context.length?this.context[0].oLoadedState:null});r("state.save()",function(){return this.iterator("table",function(a){ya(a)})});m.versionCheck=m.fnVersionCheck=function(a){for(var b=m.version.split("."),a=a.split("."),c,e,d=0,f=a.length;d<f;d++)if(c=parseInt(b[d],10)||0,e=parseInt(a[d],10)||0,c!==e)return c>e;return!0};m.isDataTable=m.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;h.each(m.settings,
-function(a,d){var f=d.nScrollHead?h("table",d.nScrollHead)[0]:null,g=d.nScrollFoot?h("table",d.nScrollFoot)[0]:null;if(d.nTable===b||f===b||g===b)c=!0});return c};m.tables=m.fnTables=function(a){return h.map(m.settings,function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable})};m.util={throttle:ua,escapeRegex:va};m.camelToHungarian=H;r("$()",function(a,b){var c=this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a,
-b){r(b+"()",function(){var a=Array.prototype.slice.call(arguments);a[0].match(/\.dt\b/)||(a[0]+=".dt");var e=h(this.tables().nodes());e[b].apply(e,a);return this})});r("clear()",function(){return this.iterator("table",function(a){oa(a)})});r("settings()",function(){return new t(this.context,this.context)});r("init()",function(){var a=this.context;return a.length?a[0].oInit:null});r("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});r("destroy()",
-function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,e=b.oClasses,d=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(d),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),q;b.bDestroying=!0;w(b,"aoDestroyCallback","destroy",[b]);a||(new t(b)).columns().visible(!0);k.unbind(".DT").find(":not(tbody *)").unbind(".DT");h(Ea).unbind(".DT-"+b.sInstance);d!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&d!=j.parentNode&&(i.children("tfoot").detach(),
-i.append(j));i.detach();k.detach();b.aaSorting=[];b.aaSortingFixed=[];xa(b);h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(e.sSortable+" "+e.sSortableAsc+" "+e.sSortableDesc+" "+e.sSortableNone);b.bJUI&&(h("th span."+e.sSortIcon+", td span."+e.sSortIcon,g).detach(),h("th, td",g).each(function(){var a=h("div."+e.sSortJUIWrapper,this);h(this).append(a.contents());a.detach()}));!a&&c&&c.insertBefore(d,b.nTableReinsertBefore);f.children().detach();f.append(l);i.css("width",b.sDestroyWidth).removeClass(e.sTable);
-(q=b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%q])});c=h.inArray(b,m.settings);-1!==c&&m.settings.splice(c,1)})});h.each(["column","row","cell"],function(a,b){r(b+"s().every()",function(a){return this.iterator(b,function(e,d,f){a.call((new t(e))[b](d,f))})})});r("i18n()",function(a,b,c){var e=this.context[0],a=R(a)(e.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]:a._);return a.replace("%d",c)});m.version="1.10.7";m.settings=
-[];m.models={};m.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};m.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null};m.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",
-sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};m.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bJQueryUI:!1,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,
-fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null,
-fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},
-sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},m.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,
-sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null};W(m.defaults);m.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};W(m.defaults.column);m.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,
-bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aoColumns:[],aoHeader:[],aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],
-sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,
-bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==B(this)?1*this._iRecordsTotal:this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==B(this)?1*this._iRecordsDisplay:
-this.aiDisplay.length},fnDisplayEnd:function(){var a=this._iDisplayLength,b=this._iDisplayStart,c=b+a,e=this.aiDisplay.length,d=this.oFeatures,f=d.bPaginate;return d.bServerSide?!1===f||-1===a?b+e:Math.min(b+a,this._iRecordsDisplay):!f||c>e||-1===a?e:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{}};m.ext=u={buttons:{},classes:{},errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},
-header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:m.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:m.version};h.extend(u,{afnFiltering:u.search,aTypes:u.type.detect,ofnSearch:u.type.search,oSort:u.type.order,afnSortData:u.order,aoFeatures:u.feature,oApi:u.internal,oStdClasses:u.classes,oPagination:u.pager});h.extend(m.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",
-sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",
-sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",sJUIHeader:"",sJUIFooter:""});var Da="",Da="",F=Da+"ui-state-default",ja=Da+"css_right ui-icon ui-icon-",Xb=Da+"fg-toolbar ui-toolbar ui-widget-header ui-helper-clearfix";h.extend(m.ext.oJUIClasses,
-m.ext.classes,{sPageButton:"fg-button ui-button "+F,sPageButtonActive:"ui-state-disabled",sPageButtonDisabled:"ui-state-disabled",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",sSortAsc:F+" sorting_asc",sSortDesc:F+" sorting_desc",sSortable:F+" sorting",sSortableAsc:F+" sorting_asc_disabled",sSortableDesc:F+" sorting_desc_disabled",sSortableNone:F+" sorting_disabled",sSortJUIAsc:ja+"triangle-1-n",sSortJUIDesc:ja+"triangle-1-s",sSortJUI:ja+"carat-2-n-s",
-sSortJUIAscAllowed:ja+"carat-1-n",sSortJUIDescAllowed:ja+"carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",sScrollHead:"dataTables_scrollHead "+F,sScrollFoot:"dataTables_scrollFoot "+F,sHeaderTH:F,sFooterTH:F,sJUIHeader:Xb+" ui-corner-tl ui-corner-tr",sJUIFooter:Xb+" ui-corner-bl ui-corner-br"});var Mb=m.ext.pager;h.extend(Mb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},simple_numbers:function(a,b){return["previous",
-Wa(a,b),"next"]},full_numbers:function(a,b){return["first","previous",Wa(a,b),"next","last"]},_numbers:Wa,numbers_length:7});h.extend(!0,m.ext.renderer,{pageButton:{_:function(a,b,c,e,d,f){var g=a.oClasses,j=a.oLanguage.oPaginate,i,k,l=0,m=function(b,e){var n,r,t,s,u=function(b){Ta(a,b.data.action,true)};n=0;for(r=e.length;n<r;n++){s=e[n];if(h.isArray(s)){t=h("<"+(s.DT_el||"div")+"/>").appendTo(b);m(t,s)}else{k=i="";switch(s){case "ellipsis":b.append('<span class="ellipsis">…</span>');break;
-case "first":i=j.sFirst;k=s+(d>0?"":" "+g.sPageButtonDisabled);break;case "previous":i=j.sPrevious;k=s+(d>0?"":" "+g.sPageButtonDisabled);break;case "next":i=j.sNext;k=s+(d<f-1?"":" "+g.sPageButtonDisabled);break;case "last":i=j.sLast;k=s+(d<f-1?"":" "+g.sPageButtonDisabled);break;default:i=s+1;k=d===s?g.sPageButtonActive:""}if(i){t=h("<a>",{"class":g.sPageButton+" "+k,"aria-controls":a.sTableId,"data-dt-idx":l,tabindex:a.iTabIndex,id:c===0&&typeof s==="string"?a.sTableId+"_"+s:null}).html(i).appendTo(b);
-Va(t,{action:s},u);l++}}}},n;try{n=h(Q.activeElement).data("dt-idx")}catch(r){}m(h(b).empty(),e);n&&h(b).find("[data-dt-idx="+n+"]").focus()}}});h.extend(m.ext.type.detect,[function(a,b){var c=b.oLanguage.sDecimal;return Za(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&(!ac.test(a)||!bc.test(a)))return null;var b=Date.parse(a);return null!==b&&!isNaN(b)||J(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return Za(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal;
-return Rb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Rb(a,c,!0)?"html-num-fmt"+c:null},function(a){return J(a)||"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);h.extend(m.ext.type.search,{html:function(a){return J(a)?a:"string"===typeof a?a.replace(Ob," ").replace(Ba,""):""},string:function(a){return J(a)?a:"string"===typeof a?a.replace(Ob," "):a}});var Aa=function(a,b,c,e){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Qb(a,b));a.replace&&(c&&(a=a.replace(c,"")),
-e&&(a=a.replace(e,"")));return 1*a};h.extend(u.type.order,{"date-pre":function(a){return Date.parse(a)||0},"html-pre":function(a){return J(a)?"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return J(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return a<b?-1:a>b?1:0},"string-desc":function(a,b){return a<b?1:a>b?-1:0}});db("");h.extend(!0,m.ext.renderer,{header:{_:function(a,b,c,e){h(a.nTable).on("order.dt.DT",function(d,
-f,g,h){if(a===f){d=c.idx;b.removeClass(c.sSortingClass+" "+e.sSortAsc+" "+e.sSortDesc).addClass(h[d]=="asc"?e.sSortAsc:h[d]=="desc"?e.sSortDesc:c.sSortingClass)}})},jqueryui:function(a,b,c,e){h("<div/>").addClass(e.sSortJUIWrapper).append(b.contents()).append(h("<span/>").addClass(e.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);h(a.nTable).on("order.dt.DT",function(d,f,g,h){if(a===f){d=c.idx;b.removeClass(e.sSortAsc+" "+e.sSortDesc).addClass(h[d]=="asc"?e.sSortAsc:h[d]=="desc"?e.sSortDesc:c.sSortingClass);
-b.find("span."+e.sSortIcon).removeClass(e.sSortJUIAsc+" "+e.sSortJUIDesc+" "+e.sSortJUI+" "+e.sSortJUIAscAllowed+" "+e.sSortJUIDescAllowed).addClass(h[d]=="asc"?e.sSortJUIAsc:h[d]=="desc"?e.sSortJUIDesc:c.sSortingClassJUI)}})}}});m.render={number:function(a,b,c,e){return{display:function(d){if("number"!==typeof d&&"string"!==typeof d)return d;var f=0>d?"-":"",d=Math.abs(parseFloat(d)),g=parseInt(d,10),d=c?b+(d-g).toFixed(c).substring(2):"";return f+(e||"")+g.toString().replace(/\B(?=(\d{3})+(?!\d))/g,
-a)+d}}}};h.extend(m.ext.internal,{_fnExternApiFunc:Nb,_fnBuildAjax:ra,_fnAjaxUpdate:kb,_fnAjaxParameters:tb,_fnAjaxUpdateDraw:ub,_fnAjaxDataSrc:sa,_fnAddColumn:Fa,_fnColumnOptions:ka,_fnAdjustColumnSizing:X,_fnVisibleToColumnIndex:la,_fnColumnIndexToVisible:$,_fnVisbleColumns:aa,_fnGetColumns:Z,_fnColumnTypes:Ha,_fnApplyColumnDefs:ib,_fnHungarianMap:W,_fnCamelToHungarian:H,_fnLanguageCompat:P,_fnBrowserDetect:gb,_fnAddData:K,_fnAddTr:ma,_fnNodeToDataIndex:function(a,b){return b._DT_RowIndex!==k?b._DT_RowIndex:
-null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:x,_fnSetCellData:Ia,_fnSplitObjNotation:Ka,_fnGetObjectDataFn:R,_fnSetObjectDataFn:S,_fnGetDataMaster:La,_fnClearTable:oa,_fnDeleteIndex:pa,_fnInvalidate:ca,_fnGetRowElements:na,_fnCreateTr:Ja,_fnBuildHead:jb,_fnDrawHead:ea,_fnDraw:M,_fnReDraw:N,_fnAddOptionsHtml:mb,_fnDetectHeader:da,_fnGetUniqueThs:qa,_fnFeatureHtmlFilter:ob,_fnFilterComplete:fa,_fnFilterCustom:xb,_fnFilterColumn:wb,_fnFilter:vb,_fnFilterCreateSearch:Qa,
-_fnEscapeRegex:va,_fnFilterData:yb,_fnFeatureHtmlInfo:rb,_fnUpdateInfo:Bb,_fnInfoMacros:Cb,_fnInitialise:ga,_fnInitComplete:ta,_fnLengthChange:Ra,_fnFeatureHtmlLength:nb,_fnFeatureHtmlPaginate:sb,_fnPageChange:Ta,_fnFeatureHtmlProcessing:pb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:qb,_fnScrollDraw:Y,_fnApplyToChildren:G,_fnCalculateColumnWidths:Ga,_fnThrottle:ua,_fnConvertToWidth:Db,_fnScrollingWidthAdjust:Fb,_fnGetWidestNode:Eb,_fnGetMaxLenString:Gb,_fnStringToCss:s,_fnScrollBarWidth:Hb,_fnSortFlatten:U,
-_fnSort:lb,_fnSortAria:Jb,_fnSortListener:Ua,_fnSortAttachListener:Oa,_fnSortingClasses:xa,_fnSortData:Ib,_fnSaveState:ya,_fnLoadState:Kb,_fnSettingsFromNode:za,_fnLog:I,_fnMap:E,_fnBindAction:Va,_fnCallbackReg:z,_fnCallbackFire:w,_fnLengthOverflow:Sa,_fnRenderer:Pa,_fnDataSource:B,_fnRowAttributes:Ma,_fnCalculateEnd:function(){}});h.fn.dataTable=m;h.fn.dataTableSettings=m.settings;h.fn.dataTableExt=m.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()};h.each(m,function(a,b){h.fn.DataTable[a]=
-b});return h.fn.dataTable};"function"===typeof define&&define.amd?define("datatables",["jquery"],P):"object"===typeof exports?module.exports=P(require("jquery")):jQuery&&!jQuery.fn.dataTable&&P(jQuery)})(window,document);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDDS-755. ContainerInfo and
ContainerReplica protobuf changes. Contributed by Nanda kumar.
Posted by ae...@apache.org.
HDDS-755. ContainerInfo and ContainerReplica protobuf changes.
Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4f22b08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4f22b08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4f22b08
Branch: refs/heads/HDDS-4
Commit: e4f22b08e0d1074c315680ba20d8666be21a25db
Parents: 773f0d1
Author: Nanda kumar <na...@apache.org>
Authored: Wed Oct 31 10:29:35 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Oct 31 10:29:35 2018 +0530
----------------------------------------------------------------------
.../scm/client/ContainerOperationClient.java | 6 +--
.../hadoop/hdds/scm/client/ScmClient.java | 6 +--
.../hdds/scm/container/ContainerInfo.java | 8 +--
...rLocationProtocolClientSideTranslatorPB.java | 2 +-
.../main/proto/DatanodeContainerProtocol.proto | 27 +++++-----
.../StorageContainerLocationProtocol.proto | 4 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 4 +-
.../container/common/impl/ContainerData.java | 22 ++++----
.../common/impl/ContainerDataYaml.java | 6 +--
.../container/common/impl/HddsDispatcher.java | 13 ++---
.../container/common/interfaces/Container.java | 9 ++--
.../container/keyvalue/KeyValueContainer.java | 31 ++++++-----
.../keyvalue/KeyValueContainerData.java | 9 ++--
.../container/keyvalue/KeyValueHandler.java | 16 +++---
.../StorageContainerDatanodeProtocol.proto | 57 +++++++++++---------
.../ozone/container/common/ScmTestMock.java | 14 ++---
.../common/TestKeyValueContainerData.java | 6 +--
.../common/impl/TestContainerDataYaml.java | 8 +--
.../container/common/impl/TestContainerSet.java | 16 +++---
.../keyvalue/TestKeyValueContainer.java | 15 +++---
.../container/keyvalue/TestKeyValueHandler.java | 2 +-
.../scm/container/ContainerReportHandler.java | 2 +-
.../hdds/scm/container/SCMContainerManager.java | 23 ++++----
.../apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +-
.../org/apache/hadoop/hdds/scm/TestUtils.java | 20 +++----
.../container/TestContainerReportHandler.java | 6 ++-
.../scm/container/TestSCMContainerManager.java | 22 ++++----
.../hdds/scm/cli/container/InfoSubcommand.java | 8 ++-
.../rpc/TestCloseContainerHandlingByClient.java | 2 +-
.../rpc/TestContainerStateMachineFailures.java | 2 +-
.../common/impl/TestContainerPersistence.java | 4 +-
.../commandhandler/TestBlockDeletion.java | 10 ++--
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 2 +-
33 files changed, 199 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
index 25a71df..8c96164 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerData;
+ .ContainerDataProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ReadContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -309,7 +309,7 @@ public class ContainerOperationClient implements ScmClient {
* @throws IOException
*/
@Override
- public ContainerData readContainer(long containerID,
+ public ContainerDataProto readContainer(long containerID,
Pipeline pipeline) throws IOException {
XceiverClientSpi client = null;
try {
@@ -337,7 +337,7 @@ public class ContainerOperationClient implements ScmClient {
* @throws IOException
*/
@Override
- public ContainerData readContainer(long containerID) throws IOException {
+ public ContainerDataProto readContainer(long containerID) throws IOException {
ContainerWithPipeline info = getContainerWithPipeline(containerID);
return readContainer(containerID, info.getPipeline());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index 3d5d56c..b3c0c94 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerData;
+ .ContainerDataProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.io.Closeable;
@@ -119,7 +119,7 @@ public interface ScmClient extends Closeable {
* @return ContainerInfo
* @throws IOException
*/
- ContainerData readContainer(long containerID, Pipeline pipeline)
+ ContainerDataProto readContainer(long containerID, Pipeline pipeline)
throws IOException;
/**
@@ -128,7 +128,7 @@ public interface ScmClient extends Closeable {
* @return ContainerInfo
* @throws IOException
*/
- ContainerData readContainer(long containerID)
+ ContainerDataProto readContainer(long containerID)
throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
index 64407a7..5a9484a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
@@ -114,7 +114,7 @@ public class ContainerInfo implements Comparator<ContainerInfo>,
public ContainerInfo() {
}
- public static ContainerInfo fromProtobuf(HddsProtos.SCMContainerInfo info) {
+ public static ContainerInfo fromProtobuf(HddsProtos.ContainerInfoProto info) {
ContainerInfo.Builder builder = new ContainerInfo.Builder();
return builder.setPipelineID(
PipelineID.getFromProtobuf(info.getPipelineID()))
@@ -191,9 +191,9 @@ public class ContainerInfo implements Comparator<ContainerInfo>,
lastUsed = Time.monotonicNow();
}
- public HddsProtos.SCMContainerInfo getProtobuf() {
- HddsProtos.SCMContainerInfo.Builder builder =
- HddsProtos.SCMContainerInfo.newBuilder();
+ public HddsProtos.ContainerInfoProto getProtobuf() {
+ HddsProtos.ContainerInfoProto.Builder builder =
+ HddsProtos.ContainerInfoProto.newBuilder();
Preconditions.checkState(containerID > 0);
return builder.setContainerID(getContainerID())
.setUsedBytes(getUsedBytes())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index d19efc1..3a4fa46 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -181,7 +181,7 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
SCMListContainerResponseProto response =
rpcProxy.listContainer(NULL_RPC_CONTROLLER, request);
List<ContainerInfo> containerList = new ArrayList<>();
- for (HddsProtos.SCMContainerInfo containerInfoProto : response
+ for (HddsProtos.ContainerInfoProto containerInfoProto : response
.getContainersList()) {
containerList.add(ContainerInfo.fromProtobuf(containerInfoProto));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 318ec09..1700e23 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -158,17 +158,6 @@ message KeyValue {
optional string value = 2;
}
-/**
- * Lifecycle states of a container in Datanode.
- */
-enum ContainerLifeCycleState {
- OPEN = 1;
- CLOSING = 2;
- CLOSED = 3;
- UNHEALTHY = 4;
- INVALID = 5;
-}
-
message ContainerCommandRequestProto {
required Type cmdType = 1; // Type of the command
@@ -235,14 +224,22 @@ message ContainerCommandResponseProto {
optional GetCommittedBlockLengthResponseProto getCommittedBlockLength = 21;
}
-message ContainerData {
+message ContainerDataProto {
+ enum State {
+ OPEN = 1;
+ CLOSING = 2;
+ CLOSED = 3;
+ QUASI_CLOSED = 4;
+ UNHEALTHY = 5;
+ INVALID = 6;
+ }
required int64 containerID = 1;
repeated KeyValue metadata = 2;
optional string containerPath = 4;
optional int64 bytesUsed = 6;
optional int64 size = 7;
optional int64 blockCount = 8;
- optional ContainerLifeCycleState state = 9 [default = OPEN];
+ optional State state = 9 [default = OPEN];
optional ContainerType containerType = 10 [default = KeyValueContainer];
}
@@ -264,7 +261,7 @@ message ReadContainerRequestProto {
}
message ReadContainerResponseProto {
- optional ContainerData containerData = 1;
+ optional ContainerDataProto containerData = 1;
}
message UpdateContainerRequestProto {
@@ -287,7 +284,7 @@ message ListContainerRequestProto {
}
message ListContainerResponseProto {
- repeated ContainerData containerData = 1;
+ repeated ContainerDataProto containerData = 1;
}
message CloseContainerRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
index 49d1975..71190ac 100644
--- a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
@@ -60,7 +60,7 @@ message GetContainerRequestProto {
}
message GetContainerResponseProto {
- required SCMContainerInfo containerInfo = 1;
+ required ContainerInfoProto containerInfo = 1;
}
message GetContainerWithPipelineRequestProto {
@@ -77,7 +77,7 @@ message SCMListContainerRequestProto {
}
message SCMListContainerResponseProto {
- repeated SCMContainerInfo containers = 1;
+ repeated ContainerInfoProto containers = 1;
}
message SCMDeleteContainerRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index 62b4833..a0c6f16 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -131,7 +131,7 @@ enum LifeCycleEvent {
CLEANUP = 8;
}
-message SCMContainerInfo {
+message ContainerInfoProto {
required int64 containerID = 1;
required LifeCycleState state = 2;
optional PipelineID pipelineID = 3;
@@ -145,7 +145,7 @@ message SCMContainerInfo {
}
message ContainerWithPipeline {
- required SCMContainerInfo containerInfo = 1;
+ required ContainerInfoProto containerInfo = 1;
required Pipeline pipeline = 2;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
index efea20b..ad199f0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java
@@ -25,8 +25,8 @@ import java.util.List;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
ContainerType;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
- ContainerLifeCycleState;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerDataProto;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
@@ -65,7 +65,7 @@ public abstract class ContainerData {
private final Map<String, String> metadata;
// State of the Container
- private ContainerLifeCycleState state;
+ private ContainerDataProto.State state;
private final long maxSize;
@@ -121,7 +121,7 @@ public abstract class ContainerData {
this.containerID = containerId;
this.layOutVersion = layOutVersion;
this.metadata = new TreeMap<>();
- this.state = ContainerLifeCycleState.OPEN;
+ this.state = ContainerDataProto.State.OPEN;
this.readCount = new AtomicLong(0L);
this.readBytes = new AtomicLong(0L);
this.writeCount = new AtomicLong(0L);
@@ -158,7 +158,7 @@ public abstract class ContainerData {
* Returns the state of the container.
* @return ContainerLifeCycleState
*/
- public synchronized ContainerLifeCycleState getState() {
+ public synchronized ContainerDataProto.State getState() {
return state;
}
@@ -166,7 +166,7 @@ public abstract class ContainerData {
* Set the state of the container.
* @param state
*/
- public synchronized void setState(ContainerLifeCycleState state) {
+ public synchronized void setState(ContainerDataProto.State state) {
this.state = state;
}
@@ -222,7 +222,7 @@ public abstract class ContainerData {
* @return - boolean
*/
public synchronized boolean isOpen() {
- return ContainerLifeCycleState.OPEN == state;
+ return ContainerDataProto.State.OPEN == state;
}
/**
@@ -230,7 +230,7 @@ public abstract class ContainerData {
* @return - boolean
*/
public synchronized boolean isValid() {
- return !(ContainerLifeCycleState.INVALID == state);
+ return !(ContainerDataProto.State.INVALID == state);
}
/**
@@ -238,14 +238,14 @@ public abstract class ContainerData {
* @return - boolean
*/
public synchronized boolean isClosed() {
- return ContainerLifeCycleState.CLOSED == state;
+ return ContainerDataProto.State.CLOSED == state;
}
/**
* Marks this container as closed.
*/
public synchronized void closeContainer() {
- setState(ContainerLifeCycleState.CLOSED);
+ setState(ContainerDataProto.State.CLOSED);
}
/**
@@ -431,5 +431,5 @@ public abstract class ContainerData {
*
* @return Protocol Buffer Message
*/
- public abstract ContainerProtos.ContainerData getProtoBufMessage();
+ public abstract ContainerProtos.ContainerDataProto getProtoBufMessage();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
index 65262d4..af705c4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
@@ -253,13 +253,13 @@ public final class ContainerDataYaml {
String state = (String) nodes.get(OzoneConsts.STATE);
switch (state) {
case "OPEN":
- kvData.setState(ContainerProtos.ContainerLifeCycleState.OPEN);
+ kvData.setState(ContainerProtos.ContainerDataProto.State.OPEN);
break;
case "CLOSING":
- kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSING);
+ kvData.setState(ContainerProtos.ContainerDataProto.State.CLOSING);
break;
case "CLOSED":
- kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSED);
+ kvData.setState(ContainerProtos.ContainerDataProto.State.CLOSED);
break;
default:
throw new IllegalStateException("Unexpected " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 1849841..6ab5b28 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -24,6 +24,8 @@ import com.google.common.collect.Maps;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerDataProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerAction;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
@@ -40,8 +42,6 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerType;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerLifeCycleState;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -156,9 +156,9 @@ public class HddsDispatcher implements ContainerDispatcher {
// which has failed, so the container is marked unhealthy right here.
// Once container is marked unhealthy, all the subsequent write
// transactions will fail with UNHEALTHY_CONTAINER exception.
- if (container.getContainerState() == ContainerLifeCycleState.OPEN) {
+ if (container.getContainerState() == ContainerDataProto.State.OPEN) {
container.getContainerData()
- .setState(ContainerLifeCycleState.UNHEALTHY);
+ .setState(ContainerDataProto.State.UNHEALTHY);
sendCloseContainerActionIfNeeded(container);
}
}
@@ -191,7 +191,7 @@ public class HddsDispatcher implements ContainerDispatcher {
private boolean isContainerFull(Container container) {
boolean isOpen = Optional.ofNullable(container)
- .map(cont -> cont.getContainerState() == ContainerLifeCycleState.OPEN)
+ .map(cont -> cont.getContainerState() == ContainerDataProto.State.OPEN)
.orElse(Boolean.FALSE);
if (isOpen) {
ContainerData containerData = container.getContainerData();
@@ -205,7 +205,8 @@ public class HddsDispatcher implements ContainerDispatcher {
private boolean isContainerUnhealthy(Container container) {
return Optional.ofNullable(container).map(
- cont -> (cont.getContainerState() == ContainerLifeCycleState.UNHEALTHY))
+ cont -> (cont.getContainerState() ==
+ ContainerDataProto.State.UNHEALTHY))
.orElse(Boolean.FALSE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
index dbef74c..65147cc 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
@@ -25,11 +25,10 @@ import java.io.OutputStream;
import java.util.Map;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerLifeCycleState;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdfs.util.RwLock;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -80,7 +79,7 @@ public interface Container<CONTAINERDATA extends ContainerData> extends RwLock {
* @return ContainerLifeCycleState - Container State.
* @throws StorageContainerException
*/
- ContainerLifeCycleState getContainerState();
+ ContainerProtos.ContainerDataProto.State getContainerState();
/**
* Closes a open container, if it is already closed or does not exist a
@@ -130,7 +129,7 @@ public interface Container<CONTAINERDATA extends ContainerData> extends RwLock {
/**
* Returns containerReport for the container.
*/
- StorageContainerDatanodeProtocolProtos.ContainerInfo getContainerReport()
+ ContainerReplicaProto getContainerReport()
throws StorageContainerException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 9a5c94c..b82c12f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -30,13 +30,11 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerLifeCycleState;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerType;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos;
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
import org.apache.hadoop.io.nativeio.NativeIO;
@@ -284,7 +282,7 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
} catch (StorageContainerException ex) {
// Failed to update .container file. Reset the state to CLOSING
- containerData.setState(ContainerLifeCycleState.CLOSING);
+ containerData.setState(ContainerProtos.ContainerDataProto.State.CLOSING);
throw ex;
} finally {
writeUnlock();
@@ -309,7 +307,7 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
}
@Override
- public ContainerLifeCycleState getContainerState() {
+ public ContainerProtos.ContainerDataProto.State getContainerState() {
return containerData.getState();
}
@@ -427,7 +425,8 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
@Override
public void exportContainerData(OutputStream destination,
ContainerPacker<KeyValueContainerData> packer) throws IOException {
- if (getContainerData().getState() != ContainerLifeCycleState.CLOSED) {
+ if (getContainerData().getState() !=
+ ContainerProtos.ContainerDataProto.State.CLOSED) {
throw new IllegalStateException(
"Only closed containers could be exported: ContainerId="
+ getContainerData().getContainerID());
@@ -518,10 +517,10 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
* Returns KeyValueContainerReport for the KeyValueContainer.
*/
@Override
- public StorageContainerDatanodeProtocolProtos.ContainerInfo
- getContainerReport() throws StorageContainerException{
- StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
- StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
+ public ContainerReplicaProto getContainerReport()
+ throws StorageContainerException {
+ ContainerReplicaProto.Builder ciBuilder =
+ ContainerReplicaProto.newBuilder();
ciBuilder.setContainerID(containerData.getContainerID())
.setReadCount(containerData.getReadCount())
.setWriteCount(containerData.getWriteCount())
@@ -540,18 +539,18 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
* @return LifeCycle State of the container in HddsProtos format
* @throws StorageContainerException
*/
- private HddsProtos.LifeCycleState getHddsState()
+ private ContainerReplicaProto.State getHddsState()
throws StorageContainerException {
- HddsProtos.LifeCycleState state;
+ ContainerReplicaProto.State state;
switch (containerData.getState()) {
case OPEN:
- state = HddsProtos.LifeCycleState.OPEN;
+ state = ContainerReplicaProto.State.OPEN;
break;
case CLOSING:
- state = HddsProtos.LifeCycleState.CLOSING;
+ state = ContainerReplicaProto.State.CLOSING;
break;
case CLOSED:
- state = HddsProtos.LifeCycleState.CLOSED;
+ state = ContainerReplicaProto.State.CLOSED;
break;
default:
throw new StorageContainerException("Invalid Container state found: " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
index 9ea84c2..f114d34 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
@@ -24,6 +24,8 @@ import java.util.Collections;
import org.apache.hadoop.conf.StorageSize;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerDataProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -245,9 +247,8 @@ public class KeyValueContainerData extends ContainerData {
*
* @return Protocol Buffer Message
*/
- public ContainerProtos.ContainerData getProtoBufMessage() {
- ContainerProtos.ContainerData.Builder builder = ContainerProtos
- .ContainerData.newBuilder();
+ public ContainerDataProto getProtoBufMessage() {
+ ContainerDataProto.Builder builder = ContainerDataProto.newBuilder();
builder.setContainerID(this.getContainerID());
builder.setContainerPath(this.getMetadataPath());
builder.setState(this.getState());
@@ -282,7 +283,7 @@ public class KeyValueContainerData extends ContainerData {
*/
@VisibleForTesting
public static KeyValueContainerData getFromProtoBuf(
- ContainerProtos.ContainerData protoData) throws IOException {
+ ContainerDataProto protoData) throws IOException {
// TODO: Add containerMaxSize to ContainerProtos.ContainerData
StorageSize storageSize = StorageSize.parse(
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index b0bc08b..7aaa5e6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -32,12 +32,12 @@ import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerDataProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerLifeCycleState;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.GetSmallFileRequestProto;
@@ -385,13 +385,13 @@ public class KeyValueHandler extends Handler {
}
long containerID = kvContainer.getContainerData().getContainerID();
- ContainerLifeCycleState containerState = kvContainer.getContainerState();
+ ContainerDataProto.State containerState = kvContainer.getContainerState();
try {
- if (containerState == ContainerLifeCycleState.CLOSED) {
+ if (containerState == ContainerDataProto.State .CLOSED) {
LOG.debug("Container {} is already closed.", containerID);
return ContainerUtils.getSuccessResponse(request);
- } else if (containerState == ContainerLifeCycleState.INVALID) {
+ } else if (containerState == ContainerDataProto.State .INVALID) {
LOG.debug("Invalid container data. ContainerID: {}", containerID);
throw new StorageContainerException("Invalid container data. " +
"ContainerID: " + containerID, INVALID_CONTAINER_STATE);
@@ -401,7 +401,7 @@ public class KeyValueHandler extends Handler {
// remove the container from open block map once, all the blocks
// have been committed and the container is closed
- kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSING);
+ kvData.setState(ContainerDataProto.State.CLOSING);
commitPendingBlocks(kvContainer);
kvContainer.close();
// make sure the the container open keys from BlockMap gets removed
@@ -798,9 +798,9 @@ public class KeyValueHandler extends Handler {
private void checkContainerOpen(KeyValueContainer kvContainer)
throws StorageContainerException {
- ContainerLifeCycleState containerState = kvContainer.getContainerState();
+ ContainerDataProto.State containerState = kvContainer.getContainerState();
- if (containerState == ContainerLifeCycleState.OPEN) {
+ if (containerState == ContainerDataProto.State.OPEN) {
return;
} else {
String msg = "Requested operation not allowed as ContainerState is " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index 72d48a6..4ddb7b2 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -80,10 +80,11 @@ message SCMHeartbeatRequestProto {
required DatanodeDetailsProto datanodeDetails = 1;
optional NodeReportProto nodeReport = 2;
optional ContainerReportsProto containerReport = 3;
- repeated CommandStatusReportsProto commandStatusReports = 4;
- optional ContainerActionsProto containerActions = 5;
- optional PipelineActionsProto pipelineActions = 6;
- optional PipelineReportsProto pipelineReports = 7;
+ optional IncrementalContainerReportProto incrementalContainerReport = 4;
+ repeated CommandStatusReportsProto commandStatusReports = 5;
+ optional ContainerActionsProto containerActions = 6;
+ optional PipelineActionsProto pipelineActions = 7;
+ optional PipelineReportsProto pipelineReports = 8;
}
/*
@@ -128,7 +129,34 @@ enum StorageTypeProto {
}
message ContainerReportsProto {
- repeated ContainerInfo reports = 1;
+ repeated ContainerReplicaProto reports = 1;
+}
+
+message IncrementalContainerReportProto {
+ repeated ContainerReplicaProto report = 1;
+}
+
+message ContainerReplicaProto {
+ enum State {
+ OPEN = 1;
+ CLOSING = 2;
+ CLOSED = 3;
+ QUASI_CLOSED = 4;
+ UNHEALTHY = 5;
+ INVALID = 6;
+ }
+ required int64 containerID = 1;
+ required State state = 2;
+ optional int64 size = 3;
+ optional int64 used = 4;
+ optional int64 keyCount = 5;
+ optional int64 readCount = 6;
+ optional int64 writeCount = 7;
+ optional int64 readBytes = 8;
+ optional int64 writeBytes = 9;
+ optional string finalhash = 10;
+ optional int64 deleteTransactionId = 11;
+ optional uint64 blockCommitSequenceId = 12;
}
message CommandStatusReportsProto {
@@ -200,25 +228,6 @@ message PipelineAction {
optional ClosePipelineInfo closePipeline = 2;
}
-/**
-A container report contains the following information.
-*/
-message ContainerInfo {
- required int64 containerID = 1;
- optional int64 size = 2;
- optional int64 used = 3;
- optional int64 keyCount = 4;
- // TODO: move the io count to separate message
- optional int64 readCount = 5;
- optional int64 writeCount = 6;
- optional int64 readBytes = 7;
- optional int64 writeBytes = 8;
- optional string finalhash = 9;
- optional hadoop.hdds.LifeCycleState state = 10;
- optional int64 deleteTransactionId = 11;
- optional uint64 blockCommitSequenceId = 12;
-}
-
/*
* These are commands returned by SCM for to the datanode to execute.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
index 3e45596..55fcf26 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.container.common;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
@@ -32,8 +34,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo;
-import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
@@ -66,8 +66,9 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
}
// Map of datanode to containers
- private Map<DatanodeDetails, Map<String, ContainerInfo>> nodeContainers =
- new HashMap();
+ private Map<DatanodeDetails,
+ Map<String, ContainerReplicaProto>> nodeContainers =
+ new HashMap<>();
private Map<DatanodeDetails, NodeReportProto> nodeReports = new HashMap<>();
private AtomicInteger commandStatusReport = new AtomicInteger(0);
private List<CommandStatus> cmdStatusList = new LinkedList<>();
@@ -274,7 +275,7 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
nodeContainers.put(datanode, containers);
}
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo report : reports
+ for (ContainerReplicaProto report : reports
.getReportsList()) {
containers.put(report.getContainerID(), report);
}
@@ -297,7 +298,8 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
* @return count of storage reports of a datanode
*/
public int getContainerCountsForDatanode(DatanodeDetails datanodeDetails) {
- Map<String, ContainerInfo> cr = nodeContainers.get(datanodeDetails);
+ Map<String, ContainerReplicaProto> cr =
+ nodeContainers.get(datanodeDetails);
if(cr != null) {
return cr.size();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
index f991520..824b770 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java
@@ -40,8 +40,8 @@ public class TestKeyValueContainerData {
.ContainerType.KeyValueContainer;
String path = "/tmp";
String containerDBType = "RocksDB";
- ContainerProtos.ContainerLifeCycleState state = ContainerProtos
- .ContainerLifeCycleState.CLOSED;
+ ContainerProtos.ContainerDataProto.State state =
+ ContainerProtos.ContainerDataProto.State.CLOSED;
AtomicLong val = new AtomicLong(0);
KeyValueContainerData kvData = new KeyValueContainerData(containerId,
@@ -49,7 +49,7 @@ public class TestKeyValueContainerData {
assertEquals(containerType, kvData.getContainerType());
assertEquals(containerId, kvData.getContainerID());
- assertEquals(ContainerProtos.ContainerLifeCycleState.OPEN, kvData
+ assertEquals(ContainerProtos.ContainerDataProto.State.OPEN, kvData
.getState());
assertEquals(0, kvData.getMetadata().size());
assertEquals(0, kvData.getNumPendingDeletionBlocks());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
index c7b9e0a..087d627 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
@@ -91,7 +91,7 @@ public class TestContainerDataYaml {
assertEquals("RocksDB", kvData.getContainerDBType());
assertEquals(containerFile.getParent(), kvData.getMetadataPath());
assertEquals(containerFile.getParent(), kvData.getChunksPath());
- assertEquals(ContainerProtos.ContainerLifeCycleState.OPEN, kvData
+ assertEquals(ContainerProtos.ContainerDataProto.State.OPEN, kvData
.getState());
assertEquals(1, kvData.getLayOutVersion());
assertEquals(0, kvData.getMetadata().size());
@@ -100,7 +100,7 @@ public class TestContainerDataYaml {
// Update ContainerData.
kvData.addMetadata("VOLUME", "hdfs");
kvData.addMetadata("OWNER", "ozone");
- kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSED);
+ kvData.setState(ContainerProtos.ContainerDataProto.State.CLOSED);
ContainerDataYaml.createContainerFile(ContainerProtos.ContainerType
@@ -117,7 +117,7 @@ public class TestContainerDataYaml {
assertEquals("RocksDB", kvData.getContainerDBType());
assertEquals(containerFile.getParent(), kvData.getMetadataPath());
assertEquals(containerFile.getParent(), kvData.getChunksPath());
- assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED, kvData
+ assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED, kvData
.getState());
assertEquals(1, kvData.getLayOutVersion());
assertEquals(2, kvData.getMetadata().size());
@@ -161,7 +161,7 @@ public class TestContainerDataYaml {
ContainerUtils.verifyChecksum(kvData);
//Checking the Container file data is consistent or not
- assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED, kvData
+ assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED, kvData
.getState());
assertEquals("RocksDB", kvData.getContainerDBType());
assertEquals(ContainerProtos.ContainerType.KeyValueContainer, kvData
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
index af322ea..7d8e438 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
@@ -51,8 +51,8 @@ public class TestContainerSet {
public void testAddGetRemoveContainer() throws StorageContainerException {
ContainerSet containerSet = new ContainerSet();
long containerId = 100L;
- ContainerProtos.ContainerLifeCycleState state = ContainerProtos
- .ContainerLifeCycleState.CLOSED;
+ ContainerProtos.ContainerDataProto.State state = ContainerProtos
+ .ContainerDataProto.State.CLOSED;
KeyValueContainerData kvData = new KeyValueContainerData(containerId,
(long) StorageUnit.GB.toBytes(5));
@@ -101,10 +101,10 @@ public class TestContainerSet {
ContainerData containerData = kv.getContainerData();
long containerId = containerData.getContainerID();
if (containerId%2 == 0) {
- assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED,
+ assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
containerData.getState());
} else {
- assertEquals(ContainerProtos.ContainerLifeCycleState.OPEN,
+ assertEquals(ContainerProtos.ContainerDataProto.State.OPEN,
containerData.getState());
}
count++;
@@ -121,10 +121,10 @@ public class TestContainerSet {
ContainerData containerData = kv.getContainerData();
long containerId = containerData.getContainerID();
if (containerId%2 == 0) {
- assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED,
+ assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
containerData.getState());
} else {
- assertEquals(ContainerProtos.ContainerLifeCycleState.OPEN,
+ assertEquals(ContainerProtos.ContainerDataProto.State.OPEN,
containerData.getState());
}
count++;
@@ -168,9 +168,9 @@ public class TestContainerSet {
KeyValueContainerData kvData = new KeyValueContainerData(i,
(long) StorageUnit.GB.toBytes(5));
if (i%2 == 0) {
- kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSED);
+ kvData.setState(ContainerProtos.ContainerDataProto.State.CLOSED);
} else {
- kvData.setState(ContainerProtos.ContainerLifeCycleState.OPEN);
+ kvData.setState(ContainerProtos.ContainerDataProto.State.OPEN);
}
KeyValueContainer kv = new KeyValueContainer(kvData, new
OzoneConfiguration());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index bf6b8b0..8c0db4a 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -24,8 +24,6 @@ import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerLifeCycleState;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
@@ -185,7 +183,8 @@ public class TestKeyValueContainer {
keyValueContainerData = keyValueContainer
.getContainerData();
- keyValueContainerData.setState(ContainerLifeCycleState.CLOSED);
+ keyValueContainerData.setState(
+ ContainerProtos.ContainerDataProto.State.CLOSED);
int numberOfKeysToWrite = 12;
//write one few keys to check the key count after import
@@ -286,7 +285,7 @@ public class TestKeyValueContainer {
@Test
public void testDeleteContainer() throws Exception {
- keyValueContainerData.setState(ContainerProtos.ContainerLifeCycleState
+ keyValueContainerData.setState(ContainerProtos.ContainerDataProto.State
.CLOSED);
keyValueContainer = new KeyValueContainer(
keyValueContainerData, conf);
@@ -315,7 +314,7 @@ public class TestKeyValueContainer {
keyValueContainerData = keyValueContainer
.getContainerData();
- assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED,
+ assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
keyValueContainerData.getState());
//Check state in the .container file
@@ -325,7 +324,7 @@ public class TestKeyValueContainer {
keyValueContainerData = (KeyValueContainerData) ContainerDataYaml
.readContainerFile(containerFile);
- assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED,
+ assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
keyValueContainerData.getState());
}
@@ -354,8 +353,8 @@ public class TestKeyValueContainer {
@Test
public void testUpdateContainerUnsupportedRequest() throws Exception {
try {
- keyValueContainerData.setState(ContainerProtos.ContainerLifeCycleState
- .CLOSED);
+ keyValueContainerData.setState(
+ ContainerProtos.ContainerDataProto.State.CLOSED);
keyValueContainer = new KeyValueContainer(keyValueContainerData, conf);
keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
Map<String, String> metadata = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index e1904c1..dcda10b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -266,7 +266,7 @@ public class TestKeyValueHandler {
KeyValueContainerData kvData = new KeyValueContainerData(containerID,
(long) StorageUnit.GB.toBytes(1));
KeyValueContainer container = new KeyValueContainer(kvData, conf);
- kvData.setState(ContainerProtos.ContainerLifeCycleState.INVALID);
+ kvData.setState(ContainerProtos.ContainerDataProto.State.INVALID);
// Create Close container request
ContainerCommandRequestProto closeContainerRequest =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
index 5885d959..0cb2f81 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
@@ -78,7 +78,7 @@ public class ContainerReportHandler implements
Set<ContainerID> containerIds = containerReport.getReportsList().stream()
.map(StorageContainerDatanodeProtocolProtos
- .ContainerInfo::getContainerID)
+ .ContainerReplicaProto::getContainerID)
.map(ContainerID::new)
.collect(Collectors.toSet());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
index 0f980dc1..4e6f09e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
@@ -22,6 +22,7 @@ import com.google.common.primitives.Longs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.scm.block.PendingDeleteStatusList;
@@ -37,7 +38,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos;
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -144,7 +145,7 @@ public class SCMContainerManager implements ContainerManager {
.getSequentialRangeKVs(null, Integer.MAX_VALUE, null);
for (Map.Entry<byte[], byte[]> entry : range) {
ContainerInfo container = ContainerInfo.fromProtobuf(
- HddsProtos.SCMContainerInfo.PARSER.parseFrom(entry.getValue()));
+ ContainerInfoProto.PARSER.parseFrom(entry.getValue()));
Preconditions.checkNotNull(container);
containerStateManager.loadContainer(container);
if (container.isOpen()) {
@@ -452,7 +453,7 @@ public class SCMContainerManager implements ContainerManager {
SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
}
ContainerInfo containerInfo = ContainerInfo.fromProtobuf(
- HddsProtos.SCMContainerInfo.parseFrom(containerBytes));
+ HddsProtos.ContainerInfoProto.parseFrom(containerBytes));
containerInfo.updateDeleteTransactionId(entry.getValue());
batch.put(dbKey, containerInfo.getProtobuf().toByteArray());
}
@@ -507,11 +508,11 @@ public class SCMContainerManager implements ContainerManager {
@Override
public void processContainerReports(DatanodeDetails datanodeDetails,
ContainerReportsProto reports) throws IOException {
- List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
+ List<ContainerReplicaProto>
containerInfos = reports.getReportsList();
PendingDeleteStatusList pendingDeleteStatusList =
new PendingDeleteStatusList(datanodeDetails);
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo newInfo :
+ for (ContainerReplicaProto newInfo :
containerInfos) {
ContainerID id = ContainerID.valueof(newInfo.getContainerID());
ContainerReplica replica = ContainerReplica.newBuilder()
@@ -523,7 +524,7 @@ public class SCMContainerManager implements ContainerManager {
try {
containerStateManager.updateContainerReplica(id, replica);
ContainerInfo currentInfo = containerStateManager.getContainer(id);
- if (newInfo.getState() == LifeCycleState.CLOSED
+ if (newInfo.getState() == ContainerReplicaProto.State.CLOSED
&& currentInfo.getState() == LifeCycleState.CLOSING) {
currentInfo = updateContainerStateInternal(id, LifeCycleEvent.CLOSE);
if (!currentInfo.isOpen()) {
@@ -532,7 +533,7 @@ public class SCMContainerManager implements ContainerManager {
}
}
- HddsProtos.SCMContainerInfo newState =
+ ContainerInfoProto newState =
reconcileState(newInfo, currentInfo);
if (currentInfo.getDeleteTransactionId() >
@@ -567,11 +568,11 @@ public class SCMContainerManager implements ContainerManager {
* @param knownState - State inside SCM.
* @return new SCM State for this container.
*/
- private HddsProtos.SCMContainerInfo reconcileState(
- StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState,
+ private HddsProtos.ContainerInfoProto reconcileState(
+ ContainerReplicaProto datanodeState,
ContainerInfo knownState) {
- HddsProtos.SCMContainerInfo.Builder builder =
- HddsProtos.SCMContainerInfo.newBuilder();
+ HddsProtos.ContainerInfoProto.Builder builder =
+ HddsProtos.ContainerInfoProto.newBuilder();
builder.setContainerID(knownState.getContainerID())
.setPipelineID(knownState.getPipelineID().getProtobuf())
.setState(knownState.getState())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
index 9b28e1e..0c79238 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java
@@ -55,7 +55,7 @@ public final class HddsTestUtils {
*/
public static NodeRegistrationContainerReport
createNodeRegistrationContainerReport(List<ContainerInfo> dnContainers) {
- List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
+ List<StorageContainerDatanodeProtocolProtos.ContainerReplicaProto>
containers = new ArrayList<>();
dnContainers.forEach(c -> {
containers.add(TestUtils.getRandomContainerInfo(c.getContainerID()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
index 9d3ec10..66ae682 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
@@ -18,6 +18,8 @@ package org.apache.hadoop.hdds.scm;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReport;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
@@ -28,8 +30,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo;
-import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol
.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
@@ -293,7 +293,7 @@ public final class TestUtils {
*/
public static ContainerReportsProto getRandomContainerReports(
int numberOfContainers) {
- List<ContainerInfo> containerInfos = new ArrayList<>();
+ List<ContainerReplicaProto> containerInfos = new ArrayList<>();
for (int i = 0; i < numberOfContainers; i++) {
containerInfos.add(getRandomContainerInfo(i));
}
@@ -326,7 +326,7 @@ public final class TestUtils {
* @return ContainerReportsProto
*/
public static ContainerReportsProto getContainerReports(
- ContainerInfo... containerInfos) {
+ ContainerReplicaProto... containerInfos) {
return getContainerReports(Arrays.asList(containerInfos));
}
@@ -338,10 +338,10 @@ public final class TestUtils {
* @return ContainerReportsProto
*/
public static ContainerReportsProto getContainerReports(
- List<ContainerInfo> containerInfos) {
+ List<ContainerReplicaProto> containerInfos) {
ContainerReportsProto.Builder
reportsBuilder = ContainerReportsProto.newBuilder();
- for (ContainerInfo containerInfo : containerInfos) {
+ for (ContainerReplicaProto containerInfo : containerInfos) {
reportsBuilder.addReports(containerInfo);
}
return reportsBuilder.build();
@@ -354,7 +354,8 @@ public final class TestUtils {
*
* @return ContainerInfo
*/
- public static ContainerInfo getRandomContainerInfo(long containerId) {
+ public static ContainerReplicaProto getRandomContainerInfo(
+ long containerId) {
return createContainerInfo(containerId,
OzoneConsts.GB * 5,
random.nextLong(1000),
@@ -379,11 +380,12 @@ public final class TestUtils {
*
* @return ContainerInfo
*/
- public static ContainerInfo createContainerInfo(
+ public static ContainerReplicaProto createContainerInfo(
long containerId, long size, long keyCount, long bytesUsed,
long readCount, long readBytes, long writeCount, long writeBytes) {
- return ContainerInfo.newBuilder()
+ return ContainerReplicaProto.newBuilder()
.setContainerID(containerId)
+ .setState(ContainerReplicaProto.State.OPEN)
.setSize(size)
.setKeyCount(keyCount)
.setUsed(bytesUsed)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
index 861d241..a5475e2 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
@@ -191,9 +191,11 @@ public class TestContainerReportHandler implements EventPublisher {
for (long containerId : containerIds) {
org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder
+ .StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.Builder
ciBuilder = org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
+ .StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.newBuilder();
ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
.setSize(5368709120L)
.setUsed(2000000000L)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
index fa0f084..02c292c 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos;
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
@@ -244,10 +244,10 @@ public class TestSCMContainerManager {
public void testFullContainerReport() throws Exception {
ContainerInfo info = createContainer();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
- List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
+ List<ContainerReplicaProto> reports =
new ArrayList<>();
- StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
- StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
+ ContainerReplicaProto.Builder ciBuilder =
+ ContainerReplicaProto.newBuilder();
ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
.setSize(5368709120L)
.setUsed(2000000000L)
@@ -257,6 +257,7 @@ public class TestSCMContainerManager {
.setReadBytes(2000000000L)
.setWriteBytes(2000000000L)
.setContainerID(info.getContainerID())
+ .setState(ContainerReplicaProto.State.CLOSED)
.setDeleteTransactionId(0);
reports.add(ciBuilder.build());
@@ -274,14 +275,14 @@ public class TestSCMContainerManager {
updatedContainer.getNumberOfKeys());
Assert.assertEquals(2000000000L, updatedContainer.getUsedBytes());
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) {
+ for (ContainerReplicaProto c : reports) {
Assert.assertEquals(containerManager.getContainerReplicas(
ContainerID.valueof(c.getContainerID())).size(), 1);
}
containerManager.processContainerReports(TestUtils.randomDatanodeDetails(),
crBuilder.build());
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) {
+ for (ContainerReplicaProto c : reports) {
Assert.assertEquals(containerManager.getContainerReplicas(
ContainerID.valueof(c.getContainerID())).size(), 2);
}
@@ -292,10 +293,10 @@ public class TestSCMContainerManager {
ContainerInfo info1 = createContainer();
ContainerInfo info2 = createContainer();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
- List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
+ List<ContainerReplicaProto> reports =
new ArrayList<>();
- StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
- StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
+ ContainerReplicaProto.Builder ciBuilder =
+ ContainerReplicaProto.newBuilder();
long cID1 = info1.getContainerID();
long cID2 = info2.getContainerID();
ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
@@ -304,7 +305,8 @@ public class TestSCMContainerManager {
.setKeyCount(100000000L)
.setReadBytes(1000000000L)
.setWriteBytes(1000000000L)
- .setContainerID(cID1);
+ .setContainerID(cID1)
+ .setState(ContainerReplicaProto.State.CLOSED);
reports.add(ciBuilder.build());
ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea54a9")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
index bf37718..f202254 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java
@@ -23,9 +23,7 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerData;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerLifeCycleState;
+ .ContainerDataProto;
import org.apache.hadoop.hdds.scm.cli.SCMCLI;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.common.helpers
@@ -64,13 +62,13 @@ public class InfoSubcommand implements Callable<Void> {
getContainerWithPipeline(containerID);
Preconditions.checkNotNull(container, "Container cannot be null");
- ContainerData containerData = scmClient.readContainer(container
+ ContainerDataProto containerData = scmClient.readContainer(container
.getContainerInfo().getContainerID(), container.getPipeline());
// Print container report info.
LOG.info("Container id: {}", containerID);
String openStatus =
- containerData.getState() == ContainerLifeCycleState.OPEN ? "OPEN" :
+ containerData.getState() == ContainerDataProto.State.OPEN ? "OPEN" :
"CLOSED";
LOG.info("Container State: {}", openStatus);
LOG.info("Container Path: {}", containerData.getContainerPath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index 76f6f8c..d06a0bc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -464,7 +464,7 @@ public class TestCloseContainerHandlingByClient {
if (datanodes.get(0).equals(datanodeService.getDatanodeDetails())) {
datanodeService.getDatanodeStateMachine().getContainer()
.getContainerSet().getContainer(containerID).getContainerData()
- .setState(ContainerProtos.ContainerLifeCycleState.CLOSING);
+ .setState(ContainerProtos.ContainerDataProto.State.CLOSING);
}
}
dataString = fixedLengthString(keyString, (chunkSize * 1 / 2));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
index 0e593fb..b3f0be7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
@@ -161,7 +161,7 @@ public class TestContainerStateMachineFailures {
.getContainer().getContainerSet()
.getContainer(omKeyLocationInfo.getContainerID())
.getContainerState()
- == ContainerProtos.ContainerLifeCycleState.UNHEALTHY);
+ == ContainerProtos.ContainerDataProto.State.UNHEALTHY);
try {
// subsequent requests will fail with unhealthy container exception
key.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index f81ee57..c2941ed 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -268,10 +268,10 @@ public class TestContainerPersistence {
// ContainerSet#getContainerReport currently returns all containers (open
// and closed) reports.
- List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
+ List<StorageContainerDatanodeProtocolProtos.ContainerReplicaProto> reports =
containerSet.getContainerReport().getReportsList();
Assert.assertEquals(10, reports.size());
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo report :
+ for (StorageContainerDatanodeProtocolProtos.ContainerReplicaProto report :
reports) {
long actualContainerID = report.getContainerID();
Assert.assertTrue(containerIDs.remove(actualContainerID));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index 63346d2..c49a98b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo;
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
@@ -233,9 +233,11 @@ public class TestBlockDeletion {
ContainerReportsProto containerReport = dnContainerSet.getContainerReport();
ContainerReportsProto.Builder dummyReportsBuilder =
ContainerReportsProto.newBuilder();
- for (ContainerInfo containerInfo : containerReport.getReportsList()) {
+ for (ContainerReplicaProto containerInfo :
+ containerReport.getReportsList()) {
dummyReportsBuilder.addReports(
- ContainerInfo.newBuilder(containerInfo).setDeleteTransactionId(0)
+ ContainerReplicaProto.newBuilder(containerInfo)
+ .setDeleteTransactionId(0)
.build());
}
ContainerReportsProto dummyReport = dummyReportsBuilder.build();
@@ -246,7 +248,7 @@ public class TestBlockDeletion {
// wait for event to be handled by event handler
Thread.sleep(1000);
String output = logCapturer.getOutput();
- for (ContainerInfo containerInfo : dummyReport.getReportsList()) {
+ for (ContainerReplicaProto containerInfo : dummyReport.getReportsList()) {
long containerId = containerInfo.getContainerID();
// Event should be triggered only for containers which have deleted blocks
if (containerIdsWithDeletedBlocks.contains(containerId)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f22b08/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
index c03128d..0c7b7ed 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
@@ -469,7 +469,7 @@ public class SQLCLI extends Configured implements Tool {
long containerID = Longs.fromByteArray(key);
ContainerInfo containerInfo = null;
containerInfo = ContainerInfo.fromProtobuf(
- HddsProtos.SCMContainerInfo.PARSER.parseFrom(value));
+ HddsProtos.ContainerInfoProto.PARSER.parseFrom(value));
Preconditions.checkNotNull(containerInfo);
try {
//TODO: include container state to sqllite schema
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDDS-751. Replace usage of Guava
Optional with Java Optional.
Posted by ae...@apache.org.
HDDS-751. Replace usage of Guava Optional with Java Optional.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d16d5f73
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d16d5f73
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d16d5f73
Branch: refs/heads/HDDS-4
Commit: d16d5f730e9d139d3e026805f21ac2c9b0bbb98b
Parents: 8fe85af
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri Nov 2 10:50:32 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Fri Nov 2 10:50:32 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdds/HddsUtils.java | 29 +++++++++-----------
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 17 ++++++------
.../hadoop/hdds/server/BaseHttpServer.java | 9 ++++--
.../java/org/apache/hadoop/ozone/OmUtils.java | 8 +++---
4 files changed, 32 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d16d5f73/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index 89edfdd..18637af 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdds;
-import com.google.common.base.Optional;
import com.google.common.base.Strings;
import com.google.common.net.HostAndPort;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -45,6 +44,7 @@ import java.util.Calendar;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
+import java.util.Optional;
import java.util.TimeZone;
import static org.apache.hadoop.hdfs.DFSConfigKeys
@@ -114,7 +114,7 @@ public final class HddsUtils {
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
return NetUtils.createSocketAddr(host.get() + ":" + port
- .or(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
+ .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
}
/**
@@ -162,7 +162,7 @@ public final class HddsUtils {
ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
return NetUtils.createSocketAddr(host.get() + ":" + port
- .or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
+ .orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
}
/**
@@ -186,7 +186,7 @@ public final class HddsUtils {
return hostName;
}
}
- return Optional.absent();
+ return Optional.empty();
}
/**
@@ -196,7 +196,7 @@ public final class HddsUtils {
*/
public static Optional<String> getHostName(String value) {
if ((value == null) || value.isEmpty()) {
- return Optional.absent();
+ return Optional.empty();
}
return Optional.of(HostAndPort.fromString(value).getHostText());
}
@@ -208,11 +208,11 @@ public final class HddsUtils {
*/
public static Optional<Integer> getHostPort(String value) {
if ((value == null) || value.isEmpty()) {
- return Optional.absent();
+ return Optional.empty();
}
int port = HostAndPort.fromString(value).getPortOrDefault(NO_PORT);
if (port == NO_PORT) {
- return Optional.absent();
+ return Optional.empty();
} else {
return Optional.of(port);
}
@@ -239,7 +239,7 @@ public final class HddsUtils {
return hostPort;
}
}
- return Optional.absent();
+ return Optional.empty();
}
/**
@@ -261,20 +261,17 @@ public final class HddsUtils {
+ " Null or empty address list found.");
}
- final com.google.common.base.Optional<Integer>
- defaultPort = com.google.common.base.Optional.of(ScmConfigKeys
- .OZONE_SCM_DEFAULT_PORT);
+ final Optional<Integer> defaultPort = Optional
+ .of(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT);
for (String address : names) {
- com.google.common.base.Optional<String> hostname =
- getHostName(address);
+ Optional<String> hostname = getHostName(address);
if (!hostname.isPresent()) {
throw new IllegalArgumentException("Invalid hostname for SCM: "
+ hostname);
}
- com.google.common.base.Optional<Integer> port =
- getHostPort(address);
+ Optional<Integer> port = getHostPort(address);
InetSocketAddress addr = NetUtils.createSocketAddr(hostname.get(),
- port.or(defaultPort.get()));
+ port.orElse(defaultPort.get()));
addresses.add(addr);
}
return addresses;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d16d5f73/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index 738a2ef..40b3d7a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.hdds.scm;
-import com.google.common.base.Optional;
import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
@@ -32,6 +31,7 @@ import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.HddsConfigKeys
@@ -114,7 +114,7 @@ public final class HddsServerUtil {
ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY);
InetSocketAddress addr = NetUtils.createSocketAddr(host.get() + ":" +
- port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
+ port.orElse(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
return addr;
}
@@ -135,8 +135,8 @@ public final class HddsServerUtil {
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
return NetUtils.createSocketAddr(
- host.or(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT) + ":" +
- port.or(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
+ host.orElse(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT) + ":" +
+ port.orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
}
/**
@@ -155,8 +155,9 @@ public final class HddsServerUtil {
ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
return NetUtils.createSocketAddr(
- host.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT) +
- ":" + port.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
+ host.orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT)
+ + ":"
+ + port.orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
}
/**
@@ -176,8 +177,8 @@ public final class HddsServerUtil {
ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY);
return NetUtils.createSocketAddr(
- host.or(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" +
- port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
+ host.orElse(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" +
+ port.orElse(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d16d5f73/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java
index da5d8da..2726fc3 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.hdds.server;
-import com.google.common.base.Optional;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -32,6 +31,7 @@ import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServlet;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.Optional;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
@@ -118,10 +118,13 @@ public abstract class BaseHttpServer {
final Optional<String> addresHost =
getHostNameFromConfigKeys(conf, addressKey);
- String hostName = bindHost.or(addresHost).or(bindHostDefault);
+ String hostName = bindHost.orElse(addresHost.get());
+ if (hostName == null || hostName.isEmpty()) {
+ hostName = bindHostDefault;
+ }
return NetUtils.createSocketAddr(
- hostName + ":" + addressPort.or(bindPortdefault));
+ hostName + ":" + addressPort.orElse(bindPortdefault));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d16d5f73/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index d8608be..9280648 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -20,13 +20,13 @@ package org.apache.hadoop.ozone;
import java.io.File;
import java.net.InetSocketAddress;
import java.util.Collection;
+import java.util.Optional;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.net.NetUtils;
-import com.google.common.base.Optional;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,7 +60,7 @@ public final class OmUtils {
OZONE_OM_ADDRESS_KEY);
return NetUtils.createSocketAddr(
- host.or(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
+ host.orElse(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
getOmRpcPort(conf));
}
@@ -90,7 +90,7 @@ public final class OmUtils {
// If no port number is specified then we'll just try the defaultBindPort.
final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
OZONE_OM_ADDRESS_KEY);
- return port.or(OZONE_OM_PORT_DEFAULT);
+ return port.orElse(OZONE_OM_PORT_DEFAULT);
}
public static int getOmRestPort(Configuration conf) {
@@ -98,7 +98,7 @@ public final class OmUtils {
// HTTP BindPort.
final Optional<Integer> port =
getPortNumberFromConfigKeys(conf, OZONE_OM_HTTP_ADDRESS_KEY);
- return port.or(OZONE_OM_HTTP_BIND_PORT_DEFAULT);
+ return port.orElse(OZONE_OM_HTTP_BIND_PORT_DEFAULT);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDDS-4
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 904e597,0000000..bf36699
mode 100644,000000..100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@@ -1,304 -1,0 +1,306 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
- import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
- import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
-
- import java.io.File;
- import java.io.IOException;
- import java.nio.file.Path;
- import java.nio.file.Paths;
- import java.util.Properties;
- import java.util.UUID;
- import java.util.concurrent.Callable;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmInfo;
+import org.apache.hadoop.hdds.scm.server.SCMStorage;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.security.KerberosAuthException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
- import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
++import java.io.File;
++import java.io.IOException;
++import java.net.InetAddress;
++import java.nio.file.Path;
++import java.nio.file.Paths;
++import java.util.Properties;
++import java.util.UUID;
++import java.util.concurrent.Callable;
++
++import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
++import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
++import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
++
+/**
+ * Test class to for security enabled Ozone cluster.
+ */
+@InterfaceAudience.Private
+public final class TestSecureOzoneCluster {
+
+ private Logger LOGGER = LoggerFactory
+ .getLogger(TestSecureOzoneCluster.class);
+
+ @Rule
+ public Timeout timeout = new Timeout(80000);
+
+ private MiniKdc miniKdc;
+ private OzoneConfiguration conf;
+ private File workDir;
+ private static Properties securityProperties;
+ private File scmKeytab;
+ private File spnegoKeytab;
+ private File omKeyTab;
+ private String curUser;
+ private StorageContainerManager scm;
+ private OzoneManager om;
+
+ private static String clusterId;
+ private static String scmId;
+ private static String omId;
+
+ @Before
+ public void init() {
+ try {
+ conf = new OzoneConfiguration();
+ startMiniKdc();
+ setSecureConfig(conf);
+ createCredentialsInKDC(conf, miniKdc);
+ } catch (IOException e) {
+ LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+ } catch (Exception e) {
+ LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+ }
+ }
+
+ @After
+ public void stop() {
+ try {
+ stopMiniKdc();
+ if (scm != null) {
+ scm.stop();
+ }
+ if (om != null) {
+ om.stop();
+ }
+ } catch (Exception e) {
+ LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
+ }
+ }
+
+ private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
+ throws Exception {
+ createPrincipal(scmKeytab,
+ conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
+ createPrincipal(spnegoKeytab,
+ conf.get(ScmConfigKeys
+ .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
+ conf.get(OMConfigKeys
+ .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
+ createPrincipal(omKeyTab,
+ conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
+ }
+
+ private void createPrincipal(File keytab, String... principal)
+ throws Exception {
+ miniKdc.createPrincipal(keytab, principal);
+ }
+
+ private void startMiniKdc() throws Exception {
+ workDir = GenericTestUtils
+ .getTestDir(TestSecureOzoneCluster.class.getSimpleName());
+ securityProperties = MiniKdc.createConf();
+ miniKdc = new MiniKdc(securityProperties, workDir);
+ miniKdc.start();
+ }
+
+ private void stopMiniKdc() throws Exception {
+ miniKdc.stop();
+ }
+
+ private void setSecureConfig(Configuration conf) throws IOException {
+ conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
- String host = KerberosUtil.getLocalHostName();
++ String host = InetAddress.getLocalHost().getCanonicalHostName();
+ String realm = miniKdc.getRealm();
+ curUser = UserGroupInformation.getCurrentUser()
+ .getUserName();
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ conf.set(OZONE_ADMINISTRATORS, curUser);
+
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
+ "scm/" + host + "@" + realm);
+ conf.set(ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY,
+ "HTTP_SCM/" + host + "@" + realm);
+
+ conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
+ "om/" + host + "@" + realm);
+ conf.set(OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY,
+ "HTTP_OM/" + host + "@" + realm);
+
+ scmKeytab = new File(workDir, "scm.keytab");
+ spnegoKeytab = new File(workDir, "http.keytab");
+ omKeyTab = new File(workDir, "om.keytab");
+
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ scmKeytab.getAbsolutePath());
+ conf.set(
+ ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY,
+ spnegoKeytab.getAbsolutePath());
+ conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+ omKeyTab.getAbsolutePath());
+ }
+
+ @Test
+ public void testSecureScmStartupSuccess() throws Exception {
+
+ initSCM();
+ scm = StorageContainerManager.createSCM(null, conf);
+ //Reads the SCM Info from SCM instance
+ ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
+ Assert.assertEquals(clusterId, scmInfo.getClusterId());
+ Assert.assertEquals(scmId, scmInfo.getScmId());
+ }
+
+ private void initSCM()
+ throws IOException, AuthenticationException {
+
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omId = UUID.randomUUID().toString();
+
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ Path scmPath = Paths.get(path, "scm-meta");
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
++ conf.set(OZONE_METADATA_DIRS, scmPath.toString());
+ conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
+ SCMStorage scmStore = new SCMStorage(conf);
+ scmStore.setClusterId(clusterId);
+ scmStore.setScmId(scmId);
+ // writes the version file properties
+ scmStore.initialize();
+ }
+
+ @Test
+ public void testSecureScmStartupFailure() throws Exception {
+ initSCM();
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+
+ LambdaTestUtils.intercept(IOException.class,
+ "Running in secure mode, but config doesn't have a keytab",
+ () -> {
+ StorageContainerManager.createSCM(null, conf);
+ });
+
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
+ "scm/_HOST@EXAMPLE.com");
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ "/etc/security/keytabs/scm.keytab");
+
+ testCommonKerberosFailures(
+ () -> StorageContainerManager.createSCM(null, conf));
+
+ }
+
+ private void testCommonKerberosFailures(Callable callable) throws Exception {
+ LambdaTestUtils.intercept(KerberosAuthException.class, "failure "
+ + "to login: for principal:", callable);
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "OAuth2");
+
+ LambdaTestUtils.intercept(IllegalArgumentException.class, "Invalid"
+ + " attribute value for hadoop.security.authentication of OAuth2",
+ callable);
+
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "KERBEROS_SSL");
+ LambdaTestUtils.intercept(AuthenticationException.class,
+ "KERBEROS_SSL authentication method not",
+ callable);
+ }
+
+ /**
+ * Tests the secure KSM Initialization Failure.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testSecureKsmInitializationFailure() throws Exception {
+ initSCM();
+ // Create a secure SCM instance as om client will connect to it
+ scm = StorageContainerManager.createSCM(null, conf);
+
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ OMStorage ksmStore = new OMStorage(conf);
+ ksmStore.setClusterId("testClusterId");
+ ksmStore.setScmId("testScmId");
+ // writes the version file properties
+ ksmStore.initialize();
+ conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
+ "non-existent-user@EXAMPLE.com");
+ testCommonKerberosFailures(() -> OzoneManager.createOm(null, conf));
+ }
+
+ /**
+ * Tests the secure KSM Initialization success.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testSecureKsmInitializationSuccess() throws Exception {
+ initSCM();
+ // Create a secure SCM instance as om client will connect to it
+ scm = StorageContainerManager.createSCM(null, conf);
+ LogCapturer logs = LogCapturer.captureLogs(OzoneManager.LOG);
+ GenericTestUtils
+ .setLogLevel(LoggerFactory.getLogger(OzoneManager.class.getName()),
+ org.slf4j.event.Level.INFO);
+
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ Path metaDirPath = Paths.get(path, "om-meta");
+
+ OMStorage omStore = new OMStorage(conf);
+ omStore.setClusterId("testClusterId");
+ omStore.setScmId("testScmId");
+ // writes the version file properties
+ omStore.initialize();
+ try {
+ om = OzoneManager.createOm(null, conf);
+ } catch (Exception ex) {
+ // Expects timeout failure from scmClient in KSM but KSM user login via
+ // kerberos should succeed
+ Assert.assertTrue(
+ logs.getOutput().contains("Ozone Manager login successful."));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-8897. LoadBasedRouterPolicy
throws NPE in case of sub cluster unavailability. Contributed by Bilwa S T.
Posted by ae...@apache.org.
YARN-8897. LoadBasedRouterPolicy throws NPE in case of sub cluster unavailability. Contributed by Bilwa S T.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aed836ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aed836ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aed836ef
Branch: refs/heads/HDDS-4
Commit: aed836efbff775d95899d05ff947f1048df8cf19
Parents: babc946
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 2 11:27:11 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Nov 2 11:27:11 2018 -0700
----------------------------------------------------------------------
.../policies/router/LoadBasedRouterPolicy.java | 6 +++-
.../router/TestLoadBasedRouterPolicy.java | 31 ++++++++++++++++++++
2 files changed, 36 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index 06e445b..fa5eb4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -95,7 +96,10 @@ public class LoadBasedRouterPolicy extends AbstractRouterPolicy {
}
}
}
-
+ if (chosen == null) {
+ throw new FederationPolicyException(
+ "Zero Active Subcluster with weight 1.");
+ }
return chosen.toId();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index dc8f99b..58f1b99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
+import static org.junit.Assert.fail;
+
import java.util.HashMap;
import java.util.Map;
@@ -103,4 +105,33 @@ public class TestLoadBasedRouterPolicy extends BaseRouterPoliciesTest {
Assert.assertEquals("sc05", chosen.getId());
}
+ @Test
+ public void testIfNoSubclustersWithWeightOne() {
+ setPolicy(new LoadBasedRouterPolicy());
+ setPolicyInfo(new WeightedPolicyInfo());
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+ // update subcluster with weight 0
+ SubClusterIdInfo sc = new SubClusterIdInfo(String.format("sc%02d", 0));
+ SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance(
+ sc.toId(), null, null, null, null, -1, SubClusterState.SC_RUNNING, -1,
+ generateClusterMetricsInfo(0));
+ getActiveSubclusters().clear();
+ getActiveSubclusters().put(sc.toId(), federationSubClusterInfo);
+ routerWeights.put(sc, 0.0f);
+ amrmWeights.put(sc, 0.0f);
+ getPolicyInfo().setRouterPolicyWeights(routerWeights);
+ getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
+
+ try {
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(), getActiveSubclusters());
+ ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext(), null);
+ fail();
+ } catch (YarnException ex) {
+ Assert.assertTrue(
+ ex.getMessage().contains("Zero Active Subcluster with weight 1"));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDFS-14049.
TestHttpFSServerWebServer fails on Windows because of missing winutils.exe.
Contributed by Inigo Goiri.
Posted by ae...@apache.org.
HDFS-14049. TestHttpFSServerWebServer fails on Windows because of missing winutils.exe. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e3df75e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e3df75e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e3df75e
Branch: refs/heads/HDDS-4
Commit: 4e3df75eb72adbab18a1d6476f228a0b504238fa
Parents: cb8d679
Author: Yiqun Lin <yq...@apache.org>
Authored: Sun Nov 4 09:15:53 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Sun Nov 4 09:15:53 2018 +0800
----------------------------------------------------------------------
.../hadoop/fs/http/server/TestHttpFSServerWebServer.java | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e3df75e/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServerWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServerWebServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServerWebServer.java
index 5250543..97d41d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServerWebServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServerWebServer.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.HadoopUsersConfTestHelper;
+import org.apache.hadoop.util.Shell;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -55,6 +56,16 @@ public class TestHttpFSServerWebServer {
confDir.mkdirs();
logsDir.mkdirs();
tempDir.mkdirs();
+
+ if (Shell.WINDOWS) {
+ File binDir = new File(homeDir, "bin");
+ binDir.mkdirs();
+ File winutils = Shell.getWinUtilsFile();
+ if (winutils.exists()) {
+ FileUtils.copyFileToDirectory(winutils, binDir);
+ }
+ }
+
System.setProperty("hadoop.home.dir", homeDir.getAbsolutePath());
System.setProperty("hadoop.log.dir", logsDir.getAbsolutePath());
System.setProperty("httpfs.home.dir", homeDir.getAbsolutePath());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDDS-796. Fix failed test
TestStorageContainerManagerHttpServer#testHttpPolicy.
Posted by ae...@apache.org.
HDDS-796. Fix failed test TestStorageContainerManagerHttpServer#testHttpPolicy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15df2e7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15df2e7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15df2e7a
Branch: refs/heads/HDDS-4
Commit: 15df2e7a7547e12e884b624d9f17ad2799d9ccf9
Parents: d43cc5d
Author: Yiqun Lin <yq...@apache.org>
Authored: Mon Nov 5 17:31:06 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Mon Nov 5 17:31:06 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdds/server/BaseHttpServer.java | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15df2e7a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java
index 2726fc3..5e7d7b8 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java
@@ -115,13 +115,10 @@ public abstract class BaseHttpServer {
final Optional<Integer> addressPort =
getPortNumberFromConfigKeys(conf, addressKey);
- final Optional<String> addresHost =
+ final Optional<String> addressHost =
getHostNameFromConfigKeys(conf, addressKey);
- String hostName = bindHost.orElse(addresHost.get());
- if (hostName == null || hostName.isEmpty()) {
- hostName = bindHostDefault;
- }
+ String hostName = bindHost.orElse(addressHost.orElse(bindHostDefault));
return NetUtils.createSocketAddr(
hostName + ":" + addressPort.orElse(bindPortdefault));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HADOOP-15885. Add base64
(urlString) support to DTUtil. Contributed by Inigo Goiri.
Posted by ae...@apache.org.
HADOOP-15885. Add base64 (urlString) support to DTUtil. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44e37b4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44e37b4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44e37b4f
Branch: refs/heads/HDDS-4
Commit: 44e37b4fd9f441becf536368a89436afcd6dede8
Parents: 6d9c18c
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 2 10:54:12 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Nov 2 10:54:12 2018 -0700
----------------------------------------------------------------------
.../hadoop/security/token/DtFileOperations.java | 28 ++++++++++++-
.../hadoop/security/token/DtUtilShell.java | 37 +++++++++++++++-
.../src/site/markdown/CommandsManual.md | 1 +
.../hadoop/security/token/TestDtUtilShell.java | 44 ++++++++++++++++++++
4 files changed, 107 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
index f154f2d..5f74f8b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
@@ -89,7 +89,7 @@ public final class DtFileOperations {
/** Add the service prefix for a local filesystem. */
private static Path fileToPath(File f) {
- return new Path("file:" + f.getAbsolutePath());
+ return new Path(f.toURI().toString());
}
/** Write out a Credentials object as a local file.
@@ -294,4 +294,30 @@ public final class DtFileOperations {
}
doFormattedWrite(tokenFile, fileFormat, creds, conf);
}
+
+ /** Import a token from a base64 encoding into the local filesystem.
+ * @param tokenFile A local File object.
+ * @param fileFormat A string equal to FORMAT_PB or FORMAT_JAVA, for output.
+ * @param alias overwrite Service field of fetched token with this text.
+ * @param base64 urlString Encoding of the token to import.
+ * @param conf Configuration object passed along.
+ * @throws IOException Error to import the token into the file.
+ */
+ public static void importTokenFile(File tokenFile, String fileFormat,
+ Text alias, String base64, Configuration conf)
+ throws IOException {
+
+ Credentials creds = tokenFile.exists() ?
+ Credentials.readTokenStorageFile(tokenFile, conf) : new Credentials();
+
+ Token<TokenIdentifier> token = new Token<>();
+ token.decodeFromUrlString(base64);
+ if (alias != null) {
+ token.setService(alias);
+ }
+ creds.addToken(token.getService(), token);
+ LOG.info("Add token with service {}", token.getService());
+
+ doFormattedWrite(tokenFile, fileFormat, creds, conf);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
index 88db34f..bc2d1b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
@@ -55,6 +55,7 @@ public class DtUtilShell extends CommandShell {
private static final String CANCEL = "cancel";
private static final String REMOVE = "remove";
private static final String RENEW = "renew";
+ private static final String IMPORT = "import";
private static final String RENEWER = "-renewer";
private static final String SERVICE = "-service";
private static final String ALIAS = "-alias";
@@ -138,6 +139,8 @@ public class DtUtilShell extends CommandShell {
setSubCommand(new Remove(false));
} else if (command.equals(RENEW)) {
setSubCommand(new Renew());
+ } else if (command.equals(IMPORT)) {
+ setSubCommand(new Import(args[++i]));
}
} else if (args[i].equals(ALIAS)) {
alias = new Text(args[++i]);
@@ -176,11 +179,11 @@ public class DtUtilShell extends CommandShell {
@Override
public String getCommandUsage() {
return String.format(
- "%n%s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n%n",
+ "%n%s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n%n",
DT_USAGE, (new Print()).getUsage(), (new Get()).getUsage(),
(new Edit()).getUsage(), (new Append()).getUsage(),
(new Remove(true)).getUsage(), (new Remove(false)).getUsage(),
- (new Renew()).getUsage());
+ (new Renew()).getUsage(), (new Import()).getUsage());
}
private class Print extends SubCommand {
@@ -357,6 +360,36 @@ public class DtUtilShell extends CommandShell {
}
}
+ private class Import extends SubCommand {
+ public static final String IMPORT_USAGE =
+ "dtutil import <base64> [-alias <alias>] " +
+ FORMAT_SUBSTRING + " filename";
+
+ private String base64 = null;
+
+ Import() { }
+
+ Import(String arg) {
+ base64 = arg;
+ }
+
+ @Override
+ public boolean validate() {
+ return true;
+ }
+
+ @Override
+ public void execute() throws Exception {
+ DtFileOperations.importTokenFile(
+ firstFile, format, alias, base64, getConf());
+ }
+
+ @Override
+ public String getUsage() {
+ return IMPORT_USAGE;
+ }
+ }
+
public static void main(String[] args) throws Exception {
System.exit(ToolRunner.run(new Configuration(), new DtUtilShell(), args));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index ce904c5..f39a92d 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -172,6 +172,7 @@ For every subcommand that connects to a service, convenience flags are provided
| `remove -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | From each file specified, remove the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. |
| `cancel -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | Just like `remove`, except the tokens are also cancelled using the service specified in the token object. <br/> *alias* must be specified. |
| `renew -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | For each file specified, renew the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. |
+| `import` *base64* <br/> `[-alias` *alias* `]` <br/> *filename* | Import a token from a base64 token. <br/> *alias* will overwrite the service field in the token. |
### `fs`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
index 53b0d3d..08554fc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
@@ -57,6 +57,13 @@ public class TestDtUtilShell {
public static Text KIND_GET = new Text("testTokenKindGet");
public static Token<?> MOCK_TOKEN =
new Token(IDENTIFIER, PASSWORD, KIND_GET, SERVICE_GET);
+
+ private static final Text SERVICE_IMPORT =
+ new Text("testTokenServiceImport");
+ private static final Text KIND_IMPORT = new Text("testTokenKindImport");
+ private static final Token<?> IMPORT_TOKEN =
+ new Token(IDENTIFIER, PASSWORD, KIND_IMPORT, SERVICE_IMPORT);
+
static {
try {
defaultConf.set("fs.defaultFS", "file:///");
@@ -73,9 +80,11 @@ public class TestDtUtilShell {
private final Path tokenFile2 = new Path(workDir, "testPrintTokenFile2");
private final Path tokenLegacyFile = new Path(workDir, "testPrintTokenFile3");
private final Path tokenFileGet = new Path(workDir, "testGetTokenFile");
+ private final Path tokenFileImport = new Path(workDir, "testImportTokenFile");
private final String tokenFilename = tokenFile.toString();
private final String tokenFilename2 = tokenFile2.toString();
private final String tokenFilenameGet = tokenFileGet.toString();
+ private final String tokenFilenameImport = tokenFileImport.toString();
private String[] args = null;
private DtUtilShell dt = null;
private int rc = 0;
@@ -283,4 +292,39 @@ public class TestDtUtilShell {
spyCreds.readTokenStorageStream(in);
Mockito.verify(spyCreds, Mockito.never()).readFields(in);
}
+
+ @Test
+ public void testImport() throws Exception {
+ String base64 = IMPORT_TOKEN.encodeToUrlString();
+ args = new String[] {"import", base64, tokenFilenameImport};
+ rc = dt.run(args);
+ assertEquals("test simple import print old exit code", 0, rc);
+
+ args = new String[] {"print", tokenFilenameImport};
+ rc = dt.run(args);
+ assertEquals("test simple import print old exit code", 0, rc);
+ assertTrue("test print after import output:\n" + outContent,
+ outContent.toString().contains(KIND_IMPORT.toString()));
+ assertTrue("test print after import output:\n" + outContent,
+ outContent.toString().contains(SERVICE_IMPORT.toString()));
+ assertTrue("test print after simple import output:\n" + outContent,
+ outContent.toString().contains(base64));
+ }
+
+ @Test
+ public void testImportWithAliasFlag() throws Exception {
+ String base64 = IMPORT_TOKEN.encodeToUrlString();
+ args = new String[] {"import", base64, "-alias", alias,
+ tokenFilenameImport};
+ rc = dt.run(args);
+ assertEquals("test import with alias print old exit code", 0, rc);
+
+ args = new String[] {"print", tokenFilenameImport};
+ rc = dt.run(args);
+ assertEquals("test simple import print old exit code", 0, rc);
+ assertTrue("test print after import output:\n" + outContent,
+ outContent.toString().contains(KIND_IMPORT.toString()));
+ assertTrue("test print after import with alias output:\n" + outContent,
+ outContent.toString().contains(alias));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-13996. Make HttpFS' ACLs
RegEx configurable. Contributed by Siyao Meng.
Posted by ae...@apache.org.
HDFS-13996. Make HttpFS' ACLs RegEx configurable. Contributed by Siyao Meng.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fe85af6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fe85af6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fe85af6
Branch: refs/heads/HDDS-4
Commit: 8fe85af63b37f2f61269e8719e5b6287f30bb0b3
Parents: d98b881
Author: Siyao Meng <sm...@cloudera.com>
Authored: Thu Nov 1 17:36:18 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Nov 1 17:36:50 2018 -0700
----------------------------------------------------------------------
.../http/server/HttpFSParametersProvider.java | 10 ++-
.../fs/http/client/BaseTestHttpFSWith.java | 60 ++++++++++++++++++
.../hadoop/fs/http/server/TestHttpFSServer.java | 65 +++++++++++++++++++-
.../org/apache/hadoop/test/TestHdfsHelper.java | 24 +++++++-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++
5 files changed, 156 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fe85af6/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
index 754ae2b..b13e6d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.fs.XAttrCodec;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.http.client.HttpFSFileSystem;
import org.apache.hadoop.fs.http.client.HttpFSFileSystem.Operation;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.lib.service.FileSystemAccess;
import org.apache.hadoop.lib.wsrs.BooleanParam;
import org.apache.hadoop.lib.wsrs.EnumParam;
import org.apache.hadoop.lib.wsrs.EnumSetParam;
@@ -37,8 +39,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT;
-
/**
* HttpFS ParametersProvider.
*/
@@ -430,7 +430,11 @@ public class HttpFSParametersProvider extends ParametersProvider {
*/
public AclPermissionParam() {
super(NAME, HttpFSFileSystem.ACLSPEC_DEFAULT,
- Pattern.compile(DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
+ Pattern.compile(HttpFSServerWebApp.get()
+ .get(FileSystemAccess.class)
+ .getFileSystemConfiguration()
+ .get(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT)));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fe85af6/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index a23ca7a..4514739 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
@@ -117,6 +119,14 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
+ // For BaseTestHttpFSWith#testFileAclsCustomizedUserAndGroupNames
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
+ "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$");
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ "^(default:)?(user|group|mask|other):" +
+ "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?" +
+ "(user|group|mask|other):[[0-9A-Za-z_][@A-Za-z0-9._-]]*:" +
+ "([rwx-]{3})?)*$");
File hdfsSite = new File(new File(homeDir, "conf"), "hdfs-site.xml");
OutputStream os = new FileOutputStream(hdfsSite);
conf.writeXml(os);
@@ -1130,6 +1140,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
testContentSummary();
break;
case FILEACLS:
+ testFileAclsCustomizedUserAndGroupNames();
testFileAcls();
break;
case DIRACLS:
@@ -1581,4 +1592,53 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
verifyGetSnapshottableDirListing(fs, dfs);
}
}
+
+ private void testFileAclsCustomizedUserAndGroupNames() throws Exception {
+ if (isLocalFS()) {
+ return;
+ }
+
+ // Get appropriate conf from the cluster
+ MiniDFSCluster miniDFSCluster = ((TestHdfsHelper) hdfsTestHelper)
+ .getMiniDFSCluster();
+ Configuration conf = miniDFSCluster.getConfiguration(0);
+ // If we call getHttpFSFileSystem() without conf from the mini cluster,
+ // WebHDFS will be initialized with the default ACL string, causing the
+ // setAcl() later to fail. This is only an issue in the unit test.
+ FileSystem httpfs = getHttpFSFileSystem(conf);
+ if (!(httpfs instanceof WebHdfsFileSystem)
+ && !(httpfs instanceof HttpFSFileSystem)) {
+ Assert.fail(httpfs.getClass().getSimpleName() +
+ " doesn't support custom user and group name pattern. "
+ + "Only WebHdfsFileSystem and HttpFSFileSystem support it.");
+ }
+ final String aclUser = "user:123:rwx";
+ final String aclGroup = "group:foo@bar:r--";
+ final String aclSet = "user::rwx," + aclUser + ",group::r--," +
+ aclGroup + ",other::r--";
+ final String dir = "/aclFileTestCustom";
+ // Create test file
+ FileSystem proxyFs = FileSystem.get(conf);
+ proxyFs.mkdirs(new Path(dir));
+ Path path = new Path(dir, "/testACL");
+ OutputStream os = proxyFs.create(path);
+ os.write(1);
+ os.close();
+ // Set ACL
+ httpfs.setAcl(path, AclEntry.parseAclSpec(aclSet, true));
+ // Verify getAclStatus responses are the same
+ AclStatus proxyAclStat = proxyFs.getAclStatus(path);
+ AclStatus httpfsAclStat = httpfs.getAclStatus(path);
+ assertSameAcls(httpfsAclStat, proxyAclStat);
+ assertSameAcls(httpfs, proxyFs, path);
+ // Verify that custom user and group are set.
+ List<String> strEntries = new ArrayList<>();
+ for (AclEntry aclEntry : httpfsAclStat.getEntries()) {
+ strEntries.add(aclEntry.toStringStable());
+ }
+ Assert.assertTrue(strEntries.contains(aclUser));
+ Assert.assertTrue(strEntries.contains(aclGroup));
+ // Clean up
+ proxyFs.delete(new Path(dir), true);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fe85af6/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
index 8f7662f..13d2482 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.fs.http.server;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.web.JsonUtil;
+import org.apache.hadoop.lib.service.FileSystemAccess;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
@@ -199,8 +201,24 @@ public class TestHttpFSServer extends HFSTestCase {
return conf;
}
- private void createHttpFSServer(boolean addDelegationTokenAuthHandler,
- boolean sslEnabled)
+ /**
+ * Write configuration to a site file under Hadoop configuration dir.
+ */
+ private void writeConf(Configuration conf, String sitename)
+ throws Exception {
+ File homeDir = TestDirHelper.getTestDir();
+ // HDFS configuration
+ File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf");
+ Assert.assertTrue(hadoopConfDir.exists());
+
+ File siteFile = new File(hadoopConfDir, sitename);
+ OutputStream os = new FileOutputStream(siteFile);
+ conf.writeXml(os);
+ os.close();
+ }
+
+ private Server createHttpFSServer(boolean addDelegationTokenAuthHandler,
+ boolean sslEnabled)
throws Exception {
Configuration conf = createHttpFSConf(addDelegationTokenAuthHandler,
sslEnabled);
@@ -213,6 +231,7 @@ public class TestHttpFSServer extends HFSTestCase {
if (addDelegationTokenAuthHandler) {
HttpFSServerWebApp.get().setAuthority(TestJettyHelper.getAuthority());
}
+ return server;
}
private String getSignedTokenString()
@@ -895,6 +914,48 @@ public class TestHttpFSServer extends HFSTestCase {
@TestDir
@TestJetty
@TestHdfs
+ public void testCustomizedUserAndGroupNames() throws Exception {
+ // Start server with default configuration
+ Server server = createHttpFSServer(false, false);
+ final Configuration conf = HttpFSServerWebApp.get()
+ .get(FileSystemAccess.class).getFileSystemConfiguration();
+ // Change pattern config
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
+ "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$");
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ "^(default:)?(user|group|mask|other):" +
+ "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?" +
+ "(user|group|mask|other):[[0-9A-Za-z_][@A-Za-z0-9._-]]*:" +
+ "([rwx-]{3})?)*$");
+ // Save configuration to site file
+ writeConf(conf, "hdfs-site.xml");
+ // Restart the HttpFS server to apply new config
+ server.stop();
+ server.start();
+
+ final String aclUser = "user:123:rw-";
+ final String aclGroup = "group:foo@bar:r--";
+ final String aclSpec = "aclspec=user::rwx," + aclUser + ",group::rwx," +
+ aclGroup + ",other::---";
+ final String dir = "/aclFileTestCustom";
+ final String path = dir + "/test";
+ // Create test dir
+ FileSystem fs = FileSystem.get(conf);
+ fs.mkdirs(new Path(dir));
+ createWithHttp(path, null);
+ // Set ACL
+ putCmd(path, "SETACL", aclSpec);
+ // Verify ACL
+ String statusJson = getStatus(path, "GETACLSTATUS");
+ List<String> aclEntries = getAclEntries(statusJson);
+ Assert.assertTrue(aclEntries.contains(aclUser));
+ Assert.assertTrue(aclEntries.contains(aclGroup));
+ }
+
+ @Test
+ @TestDir
+ @TestJetty
+ @TestHdfs
public void testOpenOffsetLength() throws Exception {
createHttpFSServer(false, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fe85af6/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java
index 40650a5..dab3230 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.StripedFileTestUtil;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.junit.Test;
import org.junit.runners.model.FrameworkMethod;
@@ -39,6 +40,8 @@ import org.junit.runners.model.Statement;
public class TestHdfsHelper extends TestDirHelper {
+ private HdfsStatement statement;
+
@Override
@Test
public void dummy() {
@@ -54,23 +57,32 @@ public class TestHdfsHelper extends TestDirHelper {
public Statement apply(Statement statement, FrameworkMethod frameworkMethod, Object o) {
TestHdfs testHdfsAnnotation = frameworkMethod.getAnnotation(TestHdfs.class);
if (testHdfsAnnotation != null) {
- statement = new HdfsStatement(statement, frameworkMethod.getName());
+ this.statement = new HdfsStatement(statement, frameworkMethod.getName());
+ statement = this.statement;
}
return super.apply(statement, frameworkMethod, o);
}
+ public MiniDFSCluster getMiniDFSCluster() {
+ return statement.getMiniDFSCluster();
+ }
+
private static class HdfsStatement extends Statement {
private Statement statement;
private String testName;
+ private MiniDFSCluster miniHdfs = null;
public HdfsStatement(Statement statement, String testName) {
this.statement = statement;
this.testName = testName;
}
+ public MiniDFSCluster getMiniDFSCluster() {
+ return miniHdfs;
+ }
+
@Override
public void evaluate() throws Throwable {
- MiniDFSCluster miniHdfs = null;
Configuration conf = HadoopUsersConfTestHelper.getBaseConf();
if (Boolean.parseBoolean(System.getProperty(HADOOP_MINI_HDFS, "true"))) {
miniHdfs = startMiniHdfs(conf);
@@ -162,6 +174,14 @@ public class TestHdfsHelper extends TestDirHelper {
conf.set("hadoop.security.authentication", "simple");
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
+ // For BaseTestHttpFSWith#testFileAclsCustomizedUserAndGroupNames
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
+ "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$");
+ conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
+ "^(default:)?(user|group|mask|other):" +
+ "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?" +
+ "(user|group|mask|other):[[0-9A-Za-z_][@A-Za-z0-9._-]]*:" +
+ "([rwx-]{3})?)*$");
FileSystemTestHelper helper = new FileSystemTestHelper();
Path targetFile = new Path(new File(helper.getTestRootDir())
.getAbsolutePath(), "test.jks");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fe85af6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 439fbae..3628b2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -62,6 +62,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
.DFS_CHECKSUM_TYPE_KEY;
public static final String DFS_CHECKSUM_TYPE_DEFAULT =
HdfsClientConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT;
+ @Deprecated
+ public static final String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY =
+ HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY;
+ @Deprecated
public static final String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT;
public static final String DFS_WEBHDFS_NETTY_LOW_WATERMARK =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDDS-677. Create documentation for
s3 gateway to the docs. Contributed by Elek Marton.
Posted by ae...@apache.org.
HDDS-677. Create documentation for s3 gateway to the docs. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6668c19d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6668c19d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6668c19d
Branch: refs/heads/HDDS-4
Commit: 6668c19dafde530c43ccacb23de11455ae1813b5
Parents: 08bb036
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Wed Oct 31 11:48:23 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Wed Oct 31 11:48:23 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/content/S3.md | 130 +++++++++++++++++++++++++++++++++++
1 file changed, 130 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6668c19d/hadoop-ozone/docs/content/S3.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/S3.md b/hadoop-ozone/docs/content/S3.md
new file mode 100644
index 0000000..cfefaf4
--- /dev/null
+++ b/hadoop-ozone/docs/content/S3.md
@@ -0,0 +1,130 @@
+---
+title: S3
+menu:
+ main:
+ parent: Client
+ weight: 1
+---
+
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+Ozone provides S3 compatible REST interface to use the object store data with any S3 compatible tools.
+
+## Getting started
+
+S3 Gateway is a separated component which provides the S3 compatible. It should be started additional to the regular Ozone components.
+
+You can start a docker based cluster, including the S3 gateway from the release package.
+
+Go to the `compose/ozones3` directory, and start the server:
+
+```bash
+docker-compose up -d
+```
+
+You can access the S3 gateway at `http://localhost:9878`
+
+## URL Schema
+
+Ozone S3 gateway supports both the virtual-host-style URL s3 bucket addresses (eg. http://bucketname.host:9878) and the path-style addresses (eg. http://host:9878/bucketname)
+
+By default it uses the path-style addressing. To use virtual host style URLs set your main domain name in your `ozone-site.xml`:
+
+```xml
+<property>
+ <name>ozone.s3g.domain.name</name>
+ <value>s3g.internal</value>
+</property>
+```
+
+## Bucket browser
+
+Bucket's could be browsed from the browser with adding `?browser=true` to the bucket URL.
+
+For example the content of the 'testbucket' could be checked from the browser using the URL http://localhost:9878/testbucket?browser=true
+
+
+## Implemented REST endpoints
+
+Operations on S3Gateway service:
+
+Endpoint | Status |
+------------|-------------|
+GET service | implemented |
+
+Operations on Bucket:
+
+Endpoint | Status | Notes
+------------------------------------|-------------|---------------
+GET Bucket (List Objects) Version 2 | implemented |
+HEAD Bucket | implemented |
+DELETE Bucket | implemented |
+PUT Bucket (Create bucket) | implemented |
+Delete Multiple Objects (POST) | implemented |
+
+Operation on Objects:
+
+Endpoint | Status | Notes
+------------------------------------|-----------------|---------------
+PUT Object | implemented |
+GET Object | implemented | Range headers are not supported
+Multipart Uplad | not implemented |
+DELETE Object | implemented |
+HEAD Object | implemented |
+
+
+## Security
+
+Security is not yet implemented, you can *use* any AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+
+Note: Ozone has a notion for 'volumes' which is missing from the S3 Rest endpoint. Under the hood S3 bucket names are maped to Ozone 'volume/bucket' locations (depending from the given authentication information).
+
+To show the storage location of a S3 bucket, use the `ozone sh bucket path <bucketname>` command.
+
+```
+aws s3api --endpoint-url http://localhost:9878 create-bucket --bucket=bucket1
+
+ozone sh bucket path bucket1
+Volume name for S3Bucket is : s3thisisakey
+Ozone FileSystem Uri is : o3fs://bucket1.s3thisisakey
+```
+
+## Clients
+
+### AWS Cli
+
+`aws` CLI could be used with specifying the custom REST endpoint.
+
+```
+aws s3api --endpoint http://localhost:9878 create-bucket --bucket buckettest
+```
+
+Or
+
+```
+aws s3 ls --endpoint http://localhost:9878 s3://buckettest
+```
+
+### S3 Fuse driver (goofys)
+
+Goofys is a S3 FUSE driver. It could be used to any mount any Ozone bucket as posix file system:
+
+
+```
+goofys --endpoint http://localhost:9878 bucket1 /mount/bucket1
+```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HDDS-777. Fix missing jenkins
issue in s3gateway module. Contributed by Bharat Viswanadham.
Posted by ae...@apache.org.
HDDS-777. Fix missing jenkins issue in s3gateway module. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c5eb237e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c5eb237e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c5eb237e
Branch: refs/heads/HDDS-4
Commit: c5eb237e3e951e27565d40a47b6f55e7eb399f5c
Parents: 6668c19
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Wed Oct 31 19:11:12 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Wed Oct 31 19:11:12 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/s3gateway/pom.xml | 2 +-
.../apache/hadoop/ozone/s3/util/S3Consts.java | 18 ++++++++++++++++++
.../apache/hadoop/ozone/s3/util/S3utils.java | 20 +++++++++++++++++++-
3 files changed, 38 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5eb237e/hadoop-ozone/s3gateway/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml
index 52eee5d..5bc7c62 100644
--- a/hadoop-ozone/s3gateway/pom.xml
+++ b/hadoop-ozone/s3gateway/pom.xml
@@ -22,7 +22,7 @@
<version>0.4.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-ozone-s3gateway</artifactId>
- <name>Apache Hadoop Ozone S3 Gatway</name>
+ <name>Apache Hadoop Ozone S3 Gateway</name>
<packaging>jar</packaging>
<version>0.4.0-SNAPSHOT</version>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5eb237e/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 70d8a96..6ece7df 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
package org.apache.hadoop.ozone.s3.util;
import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5eb237e/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
index 8af0927..652ba7f 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
package org.apache.hadoop.ozone.s3.util;
import org.apache.commons.codec.DecoderException;
@@ -52,7 +70,7 @@ public final class S3utils {
byte[] actualKeyBytes = Hex.decodeHex(hex);
String digestActualKey = DigestUtils.sha256Hex(actualKeyBytes);
if (digest.equals(digestActualKey)) {
- return new String(actualKeyBytes);
+ return new String(actualKeyBytes, StandardCharsets.UTF_8);
} else {
OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
.INVALID_ARGUMENT, key);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-7754. [Atsv2] Update document
for running v1 and v2 TS. Contributed by Suma Shivaprasad.
Posted by ae...@apache.org.
YARN-7754. [Atsv2] Update document for running v1 and v2 TS. Contributed by Suma Shivaprasad.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/486b9a4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/486b9a4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/486b9a4a
Branch: refs/heads/HDDS-4
Commit: 486b9a4a75f413aa542338b0d866c3b490381d93
Parents: a283da2
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Oct 30 11:35:01 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Oct 30 11:35:01 2018 +0530
----------------------------------------------------------------------
.../src/site/markdown/TimelineServiceV2.md | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/486b9a4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
index 2314f30..86faf6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
@@ -333,6 +333,18 @@ that it can write data to the Apache HBase cluster you are using, or set
</property>
```
+To configure both Timeline Service 1.5 and v.2, add the following property
+
+ ```
+ <property>
+ <name>yarn.timeline-service.versions</name>
+ <value>1.5f,2.0f</value>
+ </property>
+```
+
+If the above is not configured, then it defaults to the version set in `yarn.timeline-service.version`
+
+
#### Running Timeline Service v.2
Restart the resource manager as well as the node managers to pick up the new configuration. The
collectors start within the resource manager and the node managers in an embedded manner.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDDS-754. VolumeInfo#getScmUsed
throws NPE. Contributed by Hanisha Koneru.
Posted by ae...@apache.org.
HDDS-754. VolumeInfo#getScmUsed throws NPE.
Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/773f0d15
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/773f0d15
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/773f0d15
Branch: refs/heads/HDDS-4
Commit: 773f0d1519715e3ddf77c139998cc12d7447da66
Parents: e33b61f
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 30 19:17:57 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 30 19:17:57 2018 -0700
----------------------------------------------------------------------
.../container/common/volume/VolumeInfo.java | 19 +++++++++++++++++--
.../ozone/container/common/volume/VolumeSet.java | 11 +++++++----
.../container/common/volume/TestHddsVolume.java | 9 ++++++---
.../container/common/volume/TestVolumeSet.java | 4 +++-
.../hdds/scm/pipeline/TestNodeFailure.java | 3 ++-
.../apache/hadoop/ozone/MiniOzoneCluster.java | 8 ++++++++
.../hadoop/ozone/MiniOzoneClusterImpl.java | 16 +++++++++++++++-
7 files changed, 58 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
index 62fca63..0de9f18 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
@@ -95,15 +95,30 @@ public class VolumeInfo {
this.usage = new VolumeUsage(root, b.conf);
}
- public long getCapacity() {
- return configuredCapacity < 0 ? usage.getCapacity() : configuredCapacity;
+ public long getCapacity() throws IOException {
+ if (configuredCapacity < 0) {
+ if (usage == null) {
+ throw new IOException("Volume Usage thread is not running. This error" +
+ " is usually seen during DataNode shutdown.");
+ }
+ return usage.getCapacity();
+ }
+ return configuredCapacity;
}
public long getAvailable() throws IOException {
+ if (usage == null) {
+ throw new IOException("Volume Usage thread is not running. This error " +
+ "is usually seen during DataNode shutdown.");
+ }
return usage.getAvailable();
}
public long getScmUsed() throws IOException {
+ if (usage == null) {
+ throw new IOException("Volume Usage thread is not running. This error " +
+ "is usually seen during DataNode shutdown.");
+ }
return usage.getScmUsed();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
index 5b6b823..d30dd89 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
@@ -372,18 +372,21 @@ public class VolumeSet {
for (Map.Entry<String, HddsVolume> entry : volumeMap.entrySet()) {
hddsVolume = entry.getValue();
VolumeInfo volumeInfo = hddsVolume.getVolumeInfo();
- long scmUsed = 0;
- long remaining = 0;
+ long scmUsed;
+ long remaining;
+ long capacity;
failed = false;
try {
scmUsed = volumeInfo.getScmUsed();
remaining = volumeInfo.getAvailable();
+ capacity = volumeInfo.getCapacity();
} catch (IOException ex) {
LOG.warn("Failed to get scmUsed and remaining for container " +
- "storage location {}", volumeInfo.getRootDir());
+ "storage location {}", volumeInfo.getRootDir(), ex);
// reset scmUsed and remaining if df/du failed.
scmUsed = 0;
remaining = 0;
+ capacity = 0;
failed = true;
}
@@ -392,7 +395,7 @@ public class VolumeSet {
builder.setStorageLocation(volumeInfo.getRootDir())
.setId(hddsVolume.getStorageID())
.setFailed(failed)
- .setCapacity(hddsVolume.getCapacity())
+ .setCapacity(capacity)
.setRemaining(remaining)
.setScmUsed(scmUsed)
.setStorageType(hddsVolume.getStorageType());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
index 7755345..6b46762 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
@@ -31,6 +31,7 @@ import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import java.io.File;
+import java.io.IOException;
import java.util.Properties;
import java.util.UUID;
@@ -134,12 +135,14 @@ public class TestHddsVolume {
scmUsedFile.exists());
try {
- // Volume.getAvailable() should fail with NullPointerException as usage
- // is shutdown.
+ // Volume.getAvailable() should fail with IOException
+ // as usage thread is shutdown.
volume.getAvailable();
fail("HddsVolume#shutdown test failed");
} catch (Exception ex){
- assertTrue(ex instanceof NullPointerException);
+ assertTrue(ex instanceof IOException);
+ assertTrue(ex.getMessage().contains(
+ "Volume Usage thread is not running."));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
index fca68b1..7bb8a43 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
@@ -222,8 +222,10 @@ public class TestVolumeSet {
// getAvailable() should throw null pointer exception as usage is null.
volume.getAvailable();
fail("Volume shutdown failed.");
- } catch (NullPointerException ex) {
+ } catch (IOException ex) {
// Do Nothing. Exception is expected.
+ assertTrue(ex.getMessage().contains(
+ "Volume Usage thread is not running."));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
index 9a1c705..618cd8e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
@@ -99,7 +99,7 @@ public class TestNodeFailure {
}
}
- @Test
+ @Test(timeout = 300_000L)
public void testPipelineFail() throws InterruptedException, IOException,
TimeoutException {
Assert.assertEquals(ratisContainer1.getPipeline().getPipelineState(),
@@ -118,6 +118,7 @@ public class TestNodeFailure {
pipelineManager.getPipeline(ratisContainer2.getPipeline().getId())
.getPipelineState());
// Now restart the datanode and make sure that a new pipeline is created.
+ cluster.setWaitForClusterToBeReadyTimeout(300000);
cluster.restartHddsDatanode(dnToFail, true);
ContainerWithPipeline ratisContainer3 =
containerManager.allocateContainer(RATIS, THREE, "testOwner");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 3aad7f7..15bf8d0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -67,6 +67,14 @@ public interface MiniOzoneCluster {
void waitForClusterToBeReady() throws TimeoutException, InterruptedException;
/**
+ * Sets the timeout value after which
+ * {@link MiniOzoneCluster#waitForClusterToBeReady} times out.
+ *
+ * @param timeoutInMs timeout value in milliseconds
+ */
+ void setWaitForClusterToBeReadyTimeout(int timeoutInMs);
+
+ /**
* Waits/blocks till the cluster is out of chill mode.
*
* @throws TimeoutException TimeoutException In case of timeout
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 11bc0e0..6c0f408 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -90,6 +90,9 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
private final OzoneManager ozoneManager;
private final List<HddsDatanodeService> hddsDatanodes;
+ // Timeout for the cluster to be ready
+ private int waitForClusterToBeReadyTimeout = 60000; // 1 min
+
/**
* Creates a new MiniOzoneCluster.
*
@@ -122,7 +125,18 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
isReady? "Cluster is ready" : "Waiting for cluster to be ready",
healthy, hddsDatanodes.size());
return isReady;
- }, 1000, 60 * 1000); //wait for 1 min.
+ }, 1000, waitForClusterToBeReadyTimeout);
+ }
+
+ /**
+ * Sets the timeout value after which
+ * {@link MiniOzoneClusterImpl#waitForClusterToBeReady} times out.
+ *
+ * @param timeoutInMs timeout value in milliseconds
+ */
+ @Override
+ public void setWaitForClusterToBeReadyTimeout(int timeoutInMs) {
+ waitForClusterToBeReadyTimeout = timeoutInMs;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-7225. Add queue and partition
info to RM audit log. Contributed by Eric Payne
Posted by ae...@apache.org.
YARN-7225. Add queue and partition info to RM audit log. Contributed by Eric Payne
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ab611d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ab611d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ab611d4
Branch: refs/heads/HDDS-4
Commit: 2ab611d48b7669b31bd2c9b918f47251da77d0f6
Parents: d174b91
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Thu Nov 1 14:22:00 2018 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Thu Nov 1 14:22:00 2018 -0700
----------------------------------------------------------------------
.../server/resourcemanager/ClientRMService.java | 12 ++-
.../server/resourcemanager/RMAuditLogger.java | 81 +++++++++++++++++---
.../scheduler/common/fica/FiCaSchedulerApp.java | 19 ++++-
.../scheduler/fair/FSAppAttempt.java | 5 +-
.../scheduler/fifo/FifoAppAttempt.java | 10 ++-
.../resourcemanager/TestRMAuditLogger.java | 23 +++++-
6 files changed, 129 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 8f8f43e..2da000c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -580,7 +580,8 @@ public class ClientRMService extends AbstractService implements
LOG.warn("Unable to get the current user.", ie);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
ie.getMessage(), "ClientRMService",
- "Exception in submitting application", applicationId, callerContext);
+ "Exception in submitting application", applicationId, callerContext,
+ submissionContext.getQueue());
throw RPCUtil.getRemoteException(ie);
}
@@ -603,7 +604,8 @@ public class ClientRMService extends AbstractService implements
". Flow run should be a long integer", e);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
e.getMessage(), "ClientRMService",
- "Exception in submitting application", applicationId);
+ "Exception in submitting application", applicationId,
+ submissionContext.getQueue());
throw RPCUtil.getRemoteException(e);
}
}
@@ -662,12 +664,14 @@ public class ClientRMService extends AbstractService implements
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user);
RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST,
- "ClientRMService", applicationId, callerContext);
+ "ClientRMService", applicationId, callerContext,
+ submissionContext.getQueue());
} catch (YarnException e) {
LOG.info("Exception in submitting " + applicationId, e);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
e.getMessage(), "ClientRMService",
- "Exception in submitting application", applicationId, callerContext);
+ "Exception in submitting application", applicationId, callerContext,
+ submissionContext.getQueue());
throw e;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
index ab10895..292aa8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
@@ -40,7 +40,7 @@ public class RMAuditLogger {
enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID,
CALLERCONTEXT, CALLERSIGNATURE, RESOURCE, QUEUENAME,
- INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE}
+ INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE, NODELABEL}
public static class AuditConstants {
static final String SUCCESS = "SUCCESS";
@@ -98,7 +98,7 @@ public class RMAuditLogger {
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId, Resource resource) {
return createSuccessLog(user, operation, target, appId, attemptId,
- containerId, resource, null, Server.getRemoteIp());
+ containerId, resource, null, Server.getRemoteIp(), null, null);
}
/**
@@ -124,7 +124,7 @@ public class RMAuditLogger {
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId, Resource resource, CallerContext callerContext,
- InetAddress ip) {
+ InetAddress ip, String queueName, String partition) {
StringBuilder b =
createStringBuilderForSuccessEvent(user, operation, target, ip);
if (appId != null) {
@@ -140,6 +140,12 @@ public class RMAuditLogger {
add(Keys.RESOURCE, resource.toString(), b);
}
appendCallerContext(b, callerContext);
+ if (queueName != null) {
+ add(Keys.QUEUENAME, queueName, b);
+ }
+ if (partition != null) {
+ add(Keys.NODELABEL, partition, b);
+ }
return b.toString();
}
@@ -203,6 +209,32 @@ public class RMAuditLogger {
}
/**
+ * Create a readable and parseable audit log string for a successful event.
+ *
+ * @param user User who made the service request to the ResourceManager
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ * @param appId Application Id in which operation was performed.
+ * @param containerId Container Id in which operation was performed.
+ * @param resource Resource associated with container.
+ * @param queueName Name of queue.
+ * @param partition Name of labeled partition.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logSuccess(String user, String operation, String target,
+ ApplicationId appId, ContainerId containerId, Resource resource,
+ String queueName, String partition) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, appId, null,
+ containerId, resource, null, Server.getRemoteIp(), queueName,
+ partition));
+ }
+ }
+
+ /**
* Create a general readable and parseable audit log string for a successful
* event.
*
@@ -263,12 +295,20 @@ public class RMAuditLogger {
null, null));
}
}
-
+
public static void logSuccess(String user, String operation, String target,
ApplicationId appId, CallerContext callerContext) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, null, null,
- null, callerContext, Server.getRemoteIp()));
+ null, callerContext, Server.getRemoteIp(), null, null));
+ }
+ }
+
+ public static void logSuccess(String user, String operation, String target,
+ ApplicationId appId, CallerContext callerContext, String queueName) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, appId, null, null,
+ null, callerContext, Server.getRemoteIp(), queueName, null));
}
}
@@ -296,7 +336,7 @@ public class RMAuditLogger {
ApplicationId appId, InetAddress ip) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, null, null,
- null, null, ip));
+ null, null, ip, null, null));
}
}
@@ -355,7 +395,7 @@ public class RMAuditLogger {
static String createFailureLog(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
- Resource resource, CallerContext callerContext) {
+ Resource resource, CallerContext callerContext, String queueName) {
StringBuilder b = createStringBuilderForFailureLog(user,
operation, target, description, perm);
if (appId != null) {
@@ -371,6 +411,9 @@ public class RMAuditLogger {
add(Keys.RESOURCE, resource.toString(), b);
}
appendCallerContext(b, callerContext);
+ if (queueName != null) {
+ add(Keys.QUEUENAME, queueName, b);
+ }
return b.toString();
}
@@ -381,7 +424,7 @@ public class RMAuditLogger {
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId, Resource resource) {
return createFailureLog(user, operation, perm, target, description, appId,
- attemptId, containerId, resource, null);
+ attemptId, containerId, resource, null, null);
}
/**
@@ -447,13 +490,22 @@ public class RMAuditLogger {
appId, attemptId, null, null));
}
}
-
+
public static void logFailure(String user, String operation, String perm,
String target, String description, ApplicationId appId,
CallerContext callerContext) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
- appId, null, null, null, callerContext));
+ appId, null, null, null, callerContext, null));
+ }
+ }
+
+ public static void logFailure(String user, String operation, String perm,
+ String target, String description, ApplicationId appId,
+ CallerContext callerContext, String queueName) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, perm, target, description,
+ appId, null, null, null, callerContext, queueName));
}
}
@@ -480,6 +532,15 @@ public class RMAuditLogger {
}
}
+ public static void logFailure(String user, String operation, String perm,
+ String target, String description, ApplicationId appId,
+ String queueName) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, perm, target, description,
+ appId, null, null, null, null, queueName));
+ }
+ }
+
/**
* Create a readable and parseable audit log string for a failed event.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 4bfdae9..114ee51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -218,9 +218,16 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
containersToPreempt.remove(containerId);
+ // In order to save space in the audit log, only include the partition
+ // if it is not the default partition.
+ String containerPartition = null;
+ if (partition != null && !partition.isEmpty()) {
+ containerPartition = partition;
+ }
Resource containerResource = rmContainer.getContainer().getResource();
RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER,
- "SchedulerApp", getApplicationId(), containerId, containerResource);
+ "SchedulerApp", getApplicationId(), containerId, containerResource,
+ getQueueName(), containerPartition);
// Update usage metrics
queue.getMetrics().releaseResources(partition,
@@ -646,9 +653,17 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
+ " host=" + rmContainer.getAllocatedNode().getHost()
+ " type=" + allocation.getAllocationLocalityType());
}
+ // In order to save space in the audit log, only include the partition
+ // if it is not the default partition.
+ String partition =
+ schedulerContainer.getSchedulerNode().getPartition();
+ if (partition != null && partition.isEmpty()) {
+ partition = null;
+ }
RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER,
"SchedulerApp", getApplicationId(), containerId,
- allocation.getAllocatedOrReservedResource());
+ allocation.getAllocatedOrReservedResource(), getQueueName(),
+ partition);
} else {
// If the rmContainer's state is already updated to RESERVED, this is
// a reReservation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index d9f3262..2a6657a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -162,7 +162,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resource containerResource = rmContainer.getContainer().getResource();
RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER,
- "SchedulerApp", getApplicationId(), containerId, containerResource);
+ "SchedulerApp", getApplicationId(), containerId, containerResource,
+ rmContainer.getQueueName(), null);
// Update usage metrics
queue.getMetrics().releaseResources(
@@ -479,7 +480,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER,
"SchedulerApp", getApplicationId(), container.getId(),
- container.getResource());
+ container.getResource(), getQueueName(), null);
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
index 169b98a..074d5da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
@@ -99,9 +99,17 @@ public class FifoAppAttempt extends FiCaSchedulerApp {
.getApplicationAttemptId() + " container=" + containerId + " host="
+ container.getNodeId().getHost() + " type=" + type);
}
+ // In order to save space in the audit log, only include the partition
+ // if it is not the default partition.
+ String partition = null;
+ if (appAMNodePartitionName != null &&
+ !appAMNodePartitionName.isEmpty()) {
+ partition = appAMNodePartitionName;
+ }
RMAuditLogger.logSuccess(getUser(),
RMAuditLogger.AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
- getApplicationId(), containerId, container.getResource());
+ getApplicationId(), containerId, container.getResource(),
+ getQueueName(), partition);
return rmContainer;
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab611d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
index d5f1e35..7d8eeab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
@@ -67,6 +67,7 @@ public class TestRMAuditLogger {
private static final Resource RESOURCE = mock(Resource.class);
private static final String CALLER_CONTEXT = "context";
private static final byte[] CALLER_SIGNATURE = "signature".getBytes();
+ private static final String PARTITION = "label1";
@Before
public void setUp() throws Exception {
@@ -132,6 +133,14 @@ public class TestRMAuditLogger {
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, Resource resource, InetAddress remoteIp,
RMAuditLogger.ArgsBuilder args) {
+ testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId,
+ callerContext, resource, remoteIp, args, null, null);
+ }
+
+ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+ ApplicationAttemptId attemptId, ContainerId containerId,
+ CallerContext callerContext, Resource resource, InetAddress remoteIp,
+ RMAuditLogger.ArgsBuilder args, String queueName, String partition) {
String sLog;
InetAddress tmpIp = checkIP ? remoteIp : null;
if (args != null) {
@@ -139,7 +148,8 @@ public class TestRMAuditLogger {
tmpIp, args);
} else {
sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
- attemptId, containerId, resource, callerContext, tmpIp);
+ attemptId, containerId, resource, callerContext, tmpIp, queueName,
+ partition);
}
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
@@ -177,6 +187,13 @@ public class TestRMAuditLogger {
if (args != null) {
expLog.append("\tQUEUENAME=root");
expLog.append("\tRECURSIVE=true");
+ } else {
+ if (queueName != null) {
+ expLog.append("\tQUEUENAME=" + QUEUE);
+ }
+ }
+ if (partition != null) {
+ expLog.append("\tNODELABEL=" + PARTITION);
}
assertEquals(expLog.toString(), sLog);
}
@@ -258,6 +275,8 @@ public class TestRMAuditLogger {
.append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
testSuccessLogFormatHelper(checkIP, null, null, null, null, null,
Server.getRemoteIp(), args);
+ testSuccessLogFormatHelper(checkIP, null, null, null, null, null,
+ Server.getRemoteIp(), null, QUEUE, PARTITION);
testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID);
testSuccessLogNulls(checkIP);
}
@@ -283,7 +302,7 @@ public class TestRMAuditLogger {
RMAuditLogger.ArgsBuilder args) {
String fLog = args == null ?
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
- appId, attemptId, containerId, resource, callerContext) :
+ appId, attemptId, containerId, resource, callerContext, null) :
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
args);
StringBuilder expLog = new StringBuilder();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HADOOP-15687. Credentials class
should allow access to aliases.
Posted by ae...@apache.org.
HADOOP-15687. Credentials class should allow access to aliases.
Author: Lars Francke <la...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb8d679c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb8d679c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb8d679c
Branch: refs/heads/HDDS-4
Commit: cb8d679c95642842efacc5d38ccf2a61b043c689
Parents: f84a278
Author: Lars Francke <la...@gmail.com>
Authored: Sat Nov 3 16:21:29 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Nov 3 16:21:29 2018 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/security/Credentials.java | 15 ++++++
.../apache/hadoop/security/TestCredentials.java | 57 ++++++++++----------
2 files changed, 44 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb8d679c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index 4fafa4a..4b0d889 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -31,6 +31,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -142,6 +143,13 @@ public class Credentials implements Writable {
}
/**
+ * Returns an unmodifiable version of the full map of aliases to Tokens.
+ */
+ public Map<Text, Token<? extends TokenIdentifier>> getTokenMap() {
+ return Collections.unmodifiableMap(tokenMap);
+ }
+
+ /**
* @return number of Tokens in the in-memory map
*/
public int numberOfTokens() {
@@ -192,6 +200,13 @@ public class Credentials implements Writable {
}
/**
+ * Returns an unmodifiable version of the full map of aliases to secret keys.
+ */
+ public Map<Text, byte[]> getSecretKeyMap() {
+ return Collections.unmodifiableMap(secretKeysMap);
+ }
+
+ /**
* Convenience method for reading a token storage file and loading its Tokens.
* @param filename
* @param conf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb8d679c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
index 1245c07..02ba153 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
@@ -39,8 +39,6 @@ import java.util.Collection;
import javax.crypto.KeyGenerator;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.test.GenericTestUtils;
@@ -74,6 +72,9 @@ public class TestCredentials {
Token<T> token2 = new Token();
Text service1 = new Text("service1");
Text service2 = new Text("service2");
+ Text alias1 = new Text("sometoken1");
+ Text alias2 = new Text("sometoken2");
+
Collection<Text> services = new ArrayList<Text>();
services.add(service1);
@@ -81,8 +82,8 @@ public class TestCredentials {
token1.setService(service1);
token2.setService(service2);
- ts.addToken(new Text("sometoken1"), token1);
- ts.addToken(new Text("sometoken2"), token2);
+ ts.addToken(alias1, token1);
+ ts.addToken(alias2, token2);
// create keys and put it in
final KeyGenerator kg = KeyGenerator.getInstance(DEFAULT_HMAC_ALGORITHM);
@@ -109,32 +110,32 @@ public class TestCredentials {
dis.close();
// get the tokens and compare the services
- Collection<Token<? extends TokenIdentifier>> list = ts.getAllTokens();
- assertEquals("getAllTokens should return collection of size 2",
- list.size(), 2);
- boolean foundFirst = false;
- boolean foundSecond = false;
- for (Token<? extends TokenIdentifier> token : list) {
- if (token.getService().equals(service1)) {
- foundFirst = true;
- }
- if (token.getService().equals(service2)) {
- foundSecond = true;
- }
- }
- assertTrue("Tokens for services service1 and service2 must be present",
- foundFirst && foundSecond);
+ Map<Text, Token<? extends TokenIdentifier>> tokenMap = ts.getTokenMap();
+ assertEquals("getTokenMap should return collection of size 2", 2,
+ tokenMap.size());
+ assertTrue("Token for alias " + alias1 + " must be present",
+ tokenMap.containsKey(alias1));
+ assertTrue("Token for alias " + alias2 + " must be present",
+ tokenMap.containsKey(alias2));
+ assertEquals("Token for service " + service1 + " must be present", service1,
+ tokenMap.get(alias1).getService());
+ assertEquals("Token for service " + service2 + " must be present", service2,
+ tokenMap.get(alias2).getService());
+
+
// compare secret keys
- int mapLen = m.size();
- assertEquals("wrong number of keys in the Storage",
- mapLen, ts.numberOfSecretKeys());
- for(Text a : m.keySet()) {
- byte [] kTS = ts.getSecretKey(a);
- byte [] kLocal = m.get(a);
- assertTrue("keys don't match for " + a,
- WritableComparator.compareBytes(kTS, 0, kTS.length, kLocal,
- 0, kLocal.length)==0);
+ Map<Text, byte[]> secretKeyMap = ts.getSecretKeyMap();
+ assertEquals("wrong number of keys in the Storage", m.size(),
+ ts.numberOfSecretKeys());
+
+ for (Map.Entry<Text, byte[]> entry : m.entrySet()) {
+ byte[] key = secretKeyMap.get(entry.getKey());
+ assertNotNull("Secret key for alias " + entry.getKey() + " not found",
+ key);
+ assertTrue("Keys don't match for alias " + entry.getKey(),
+ Arrays.equals(key, entry.getValue()));
}
+
tmpFileName.delete();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HADOOP-15899. Update AWS Java SDK
versions in NOTICE.txt.
Posted by ae...@apache.org.
HADOOP-15899. Update AWS Java SDK versions in NOTICE.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5e21cd6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5e21cd6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5e21cd6
Branch: refs/heads/HDDS-4
Commit: a5e21cd6a450990d00eb7515d579b51d2a0bff82
Parents: 4e3df75
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Nov 5 15:28:21 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Nov 5 15:28:21 2018 +0900
----------------------------------------------------------------------
NOTICE.txt | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5e21cd6/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 00fa375..d6e5488 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -8,10 +8,10 @@ following notices:
* Copyright 2011 FuseSource Corp. http://fusesource.com
The binary distribution of this product bundles binaries of
-AWS SDK for Java - Bundle 1.11.134,
-AWS Java SDK for AWS KMS 1.11.134,
-AWS Java SDK for Amazon S3 1.11.134,
-AWS Java SDK for AWS STS 1.11.134,
+AWS SDK for Java - Bundle 1.11.375,
+AWS Java SDK for AWS KMS 1.11.375,
+AWS Java SDK for Amazon S3 1.11.375,
+AWS Java SDK for AWS STS 1.11.375,
JMES Path Query library 1.0,
which has the following notices:
* This software includes third party software subject to the following
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-8950. Fix compilation issue
due to dependency convergence error for hbase.profile=2.0.
Posted by ae...@apache.org.
YARN-8950. Fix compilation issue due to dependency convergence error for hbase.profile=2.0.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ec4ec69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ec4ec69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ec4ec69
Branch: refs/heads/HDDS-4
Commit: 4ec4ec69711180d642c5b56cd3d3dbdf44d3c61f
Parents: db7e636
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Oct 30 11:29:58 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Oct 30 11:30:08 2018 +0530
----------------------------------------------------------------------
.../hadoop-yarn-server-timelineservice-hbase-client/pom.xml | 8 ++++++++
.../pom.xml | 8 ++++++++
2 files changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec4ec69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
index 86b2158..4225519 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
@@ -160,6 +160,14 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ </exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec4ec69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
index 4fde40c..984cac9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
@@ -147,6 +147,14 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-8854. Upgrade jquery
datatable version references to v1.10.19. Contributed by Akhil PB.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/demo_table.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/demo_table.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/demo_table.css
deleted file mode 100644
index 37b9203..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/demo_table.css
+++ /dev/null
@@ -1,538 +0,0 @@
-/*
- * File: demo_table.css
- * CVS: $Id$
- * Description: CSS descriptions for DataTables demo pages
- * Author: Allan Jardine
- * Created: Tue May 12 06:47:22 BST 2009
- * Modified: $Date$ by $Author$
- * Language: CSS
- * Project: DataTables
- *
- * Copyright 2009 Allan Jardine. All Rights Reserved.
- *
- * ***************************************************************************
- * DESCRIPTION
- *
- * The styles given here are suitable for the demos that are used with the standard DataTables
- * distribution (see www.datatables.net). You will most likely wish to modify these styles to
- * meet the layout requirements of your site.
- *
- * Common issues:
- * 'full_numbers' pagination - I use an extra selector on the body tag to ensure that there is
- * no conflict between the two pagination types. If you want to use full_numbers pagination
- * ensure that you either have "example_alt_pagination" as a body class name, or better yet,
- * modify that selector.
- * Note that the path used for Images is relative. All images are by default located in
- * ../images/ - relative to this CSS file.
- */
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * DataTables features
- */
-
-.dataTables_wrapper {
- position: relative;
- min-height: 302px;
- clear: both;
- _height: 302px;
- zoom: 1; /* Feeling sorry for IE */
-}
-
-.dataTables_processing {
- position: absolute;
- top: 50%;
- left: 50%;
- width: 250px;
- height: 30px;
- margin-left: -125px;
- margin-top: -15px;
- padding: 14px 0 2px 0;
- border: 1px solid #ddd;
- text-align: center;
- color: #999;
- font-size: 14px;
- background-color: white;
-}
-
-.dataTables_length {
- width: 40%;
- float: left;
-}
-
-.dataTables_filter {
- width: 50%;
- float: right;
- text-align: right;
-}
-
-.dataTables_info {
- width: 60%;
- float: left;
-}
-
-.dataTables_paginate {
- width: 44px;
- * width: 50px;
- float: right;
- text-align: right;
-}
-
-/* Pagination nested */
-.paginate_disabled_previous, .paginate_enabled_previous, .paginate_disabled_next, .paginate_enabled_next {
- height: 19px;
- width: 19px;
- margin-left: 3px;
- float: left;
-}
-
-.paginate_disabled_previous {
- background-image: url('../images/back_disabled.jpg');
-}
-
-.paginate_enabled_previous {
- background-image: url('../images/back_enabled.jpg');
-}
-
-.paginate_disabled_next {
- background-image: url('../images/forward_disabled.jpg');
-}
-
-.paginate_enabled_next {
- background-image: url('../images/forward_enabled.jpg');
-}
-
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * DataTables display
- */
-table.display {
- margin: 0 auto;
- clear: both;
- width: 100%;
-
- /* Note Firefox 3.5 and before have a bug with border-collapse
- * ( https://bugzilla.mozilla.org/show%5Fbug.cgi?id=155955 )
- * border-spacing: 0; is one possible option. Conditional-css.com is
- * useful for this kind of thing
- *
- * Further note IE 6/7 has problems when calculating widths with border width.
- * It subtracts one px relative to the other browsers from the first column, and
- * adds one to the end...
- *
- * If you want that effect I'd suggest setting a border-top/left on th/td's and
- * then filling in the gaps with other borders.
- */
-}
-
-table.display thead th {
- padding: 3px 18px 3px 10px;
- border-bottom: 1px solid black;
- font-weight: bold;
- cursor: pointer;
- * cursor: hand;
-}
-
-table.display tfoot th {
- padding: 3px 18px 3px 10px;
- border-top: 1px solid black;
- font-weight: bold;
-}
-
-table.display tr.heading2 td {
- border-bottom: 1px solid #aaa;
-}
-
-table.display td {
- padding: 3px 10px;
-}
-
-table.display td.center {
- text-align: center;
-}
-
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * DataTables sorting
- */
-
-.sorting_asc {
- background: url('../images/sort_asc.png') no-repeat center right;
-}
-
-.sorting_desc {
- background: url('../images/sort_desc.png') no-repeat center right;
-}
-
-.sorting {
- background: url('../images/sort_both.png') no-repeat center right;
-}
-
-.sorting_asc_disabled {
- background: url('../images/sort_asc_disabled.png') no-repeat center right;
-}
-
-.sorting_desc_disabled {
- background: url('../images/sort_desc_disabled.png') no-repeat center right;
-}
-
-
-
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * DataTables row classes
- */
-table.display tr.odd.gradeA {
- background-color: #ddffdd;
-}
-
-table.display tr.even.gradeA {
- background-color: #eeffee;
-}
-
-table.display tr.odd.gradeC {
- background-color: #ddddff;
-}
-
-table.display tr.even.gradeC {
- background-color: #eeeeff;
-}
-
-table.display tr.odd.gradeX {
- background-color: #ffdddd;
-}
-
-table.display tr.even.gradeX {
- background-color: #ffeeee;
-}
-
-table.display tr.odd.gradeU {
- background-color: #ddd;
-}
-
-table.display tr.even.gradeU {
- background-color: #eee;
-}
-
-
-tr.odd {
- background-color: #E2E4FF;
-}
-
-tr.even {
- background-color: white;
-}
-
-
-
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * Misc
- */
-.dataTables_scroll {
- clear: both;
-}
-
-.dataTables_scrollBody {
- *margin-top: -1px;
-}
-
-.top, .bottom {
- padding: 15px;
- background-color: #F5F5F5;
- border: 1px solid #CCCCCC;
-}
-
-.top .dataTables_info {
- float: none;
-}
-
-.clear {
- clear: both;
-}
-
-.dataTables_empty {
- text-align: center;
-}
-
-tfoot input {
- margin: 0.5em 0;
- width: 100%;
- color: #444;
-}
-
-tfoot input.search_init {
- color: #999;
-}
-
-td.group {
- background-color: #d1cfd0;
- border-bottom: 2px solid #A19B9E;
- border-top: 2px solid #A19B9E;
-}
-
-td.details {
- background-color: #d1cfd0;
- border: 2px solid #A19B9E;
-}
-
-
-.example_alt_pagination div.dataTables_info {
- width: 40%;
-}
-
-.paging_full_numbers {
- width: 400px;
- height: 22px;
- line-height: 22px;
-}
-
-.paging_full_numbers span.paginate_button,
- .paging_full_numbers span.paginate_active {
- border: 1px solid #aaa;
- -webkit-border-radius: 5px;
- -moz-border-radius: 5px;
- padding: 2px 5px;
- margin: 0 3px;
- cursor: pointer;
- *cursor: hand;
-}
-
-.paging_full_numbers span.paginate_button {
- background-color: #ddd;
-}
-
-.paging_full_numbers span.paginate_button:hover {
- background-color: #ccc;
-}
-
-.paging_full_numbers span.paginate_active {
- background-color: #99B3FF;
-}
-
-table.display tr.even.row_selected td {
- background-color: #B0BED9;
-}
-
-table.display tr.odd.row_selected td {
- background-color: #9FAFD1;
-}
-
-
-/*
- * Sorting classes for columns
- */
-/* For the standard odd/even */
-tr.odd td.sorting_1 {
- background-color: #D3D6FF;
-}
-
-tr.odd td.sorting_2 {
- background-color: #DADCFF;
-}
-
-tr.odd td.sorting_3 {
- background-color: #E0E2FF;
-}
-
-tr.even td.sorting_1 {
- background-color: #EAEBFF;
-}
-
-tr.even td.sorting_2 {
- background-color: #F2F3FF;
-}
-
-tr.even td.sorting_3 {
- background-color: #F9F9FF;
-}
-
-
-/* For the Conditional-CSS grading rows */
-/*
- Colour calculations (based off the main row colours)
- Level 1:
- dd > c4
- ee > d5
- Level 2:
- dd > d1
- ee > e2
- */
-tr.odd.gradeA td.sorting_1 {
- background-color: #c4ffc4;
-}
-
-tr.odd.gradeA td.sorting_2 {
- background-color: #d1ffd1;
-}
-
-tr.odd.gradeA td.sorting_3 {
- background-color: #d1ffd1;
-}
-
-tr.even.gradeA td.sorting_1 {
- background-color: #d5ffd5;
-}
-
-tr.even.gradeA td.sorting_2 {
- background-color: #e2ffe2;
-}
-
-tr.even.gradeA td.sorting_3 {
- background-color: #e2ffe2;
-}
-
-tr.odd.gradeC td.sorting_1 {
- background-color: #c4c4ff;
-}
-
-tr.odd.gradeC td.sorting_2 {
- background-color: #d1d1ff;
-}
-
-tr.odd.gradeC td.sorting_3 {
- background-color: #d1d1ff;
-}
-
-tr.even.gradeC td.sorting_1 {
- background-color: #d5d5ff;
-}
-
-tr.even.gradeC td.sorting_2 {
- background-color: #e2e2ff;
-}
-
-tr.even.gradeC td.sorting_3 {
- background-color: #e2e2ff;
-}
-
-tr.odd.gradeX td.sorting_1 {
- background-color: #ffc4c4;
-}
-
-tr.odd.gradeX td.sorting_2 {
- background-color: #ffd1d1;
-}
-
-tr.odd.gradeX td.sorting_3 {
- background-color: #ffd1d1;
-}
-
-tr.even.gradeX td.sorting_1 {
- background-color: #ffd5d5;
-}
-
-tr.even.gradeX td.sorting_2 {
- background-color: #ffe2e2;
-}
-
-tr.even.gradeX td.sorting_3 {
- background-color: #ffe2e2;
-}
-
-tr.odd.gradeU td.sorting_1 {
- background-color: #c4c4c4;
-}
-
-tr.odd.gradeU td.sorting_2 {
- background-color: #d1d1d1;
-}
-
-tr.odd.gradeU td.sorting_3 {
- background-color: #d1d1d1;
-}
-
-tr.even.gradeU td.sorting_1 {
- background-color: #d5d5d5;
-}
-
-tr.even.gradeU td.sorting_2 {
- background-color: #e2e2e2;
-}
-
-tr.even.gradeU td.sorting_3 {
- background-color: #e2e2e2;
-}
-
-
-/*
- * Row highlighting example
- */
-.ex_highlight #example tbody tr.even:hover, #example tbody tr.even td.highlighted {
- background-color: #ECFFB3;
-}
-
-.ex_highlight #example tbody tr.odd:hover, #example tbody tr.odd td.highlighted {
- background-color: #E6FF99;
-}
-
-.ex_highlight_row #example tr.even:hover {
- background-color: #ECFFB3;
-}
-
-.ex_highlight_row #example tr.even:hover td.sorting_1 {
- background-color: #DDFF75;
-}
-
-.ex_highlight_row #example tr.even:hover td.sorting_2 {
- background-color: #E7FF9E;
-}
-
-.ex_highlight_row #example tr.even:hover td.sorting_3 {
- background-color: #E2FF89;
-}
-
-.ex_highlight_row #example tr.odd:hover {
- background-color: #E6FF99;
-}
-
-.ex_highlight_row #example tr.odd:hover td.sorting_1 {
- background-color: #D6FF5C;
-}
-
-.ex_highlight_row #example tr.odd:hover td.sorting_2 {
- background-color: #E0FF84;
-}
-
-.ex_highlight_row #example tr.odd:hover td.sorting_3 {
- background-color: #DBFF70;
-}
-
-
-/*
- * KeyTable
- */
-table.KeyTable td {
- border: 3px solid transparent;
-}
-
-table.KeyTable td.focus {
- border: 3px solid #3366FF;
-}
-
-table.display tr.gradeA {
- background-color: #eeffee;
-}
-
-table.display tr.gradeC {
- background-color: #ddddff;
-}
-
-table.display tr.gradeX {
- background-color: #ffdddd;
-}
-
-table.display tr.gradeU {
- background-color: #ddd;
-}
-
-div.box {
- height: 100px;
- padding: 10px;
- overflow: auto;
- border: 1px solid #8080FF;
- background-color: #E5E5FF;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/jui-dt.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/jui-dt.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/jui-dt.css
deleted file mode 100644
index de8faea..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/jui-dt.css
+++ /dev/null
@@ -1,322 +0,0 @@
-/*
- * File: demo_table_jui.css
- * CVS: $Id$
- * Description: CSS descriptions for DataTables demo pages
- * Author: Allan Jardine
- * Created: Tue May 12 06:47:22 BST 2009
- * Modified: $Date$ by $Author$
- * Language: CSS
- * Project: DataTables
- *
- * Copyright 2009 Allan Jardine. All Rights Reserved.
- *
- * ***************************************************************************
- * DESCRIPTION
- *
- * The styles given here are suitable for the demos that are used with the standard DataTables
- * distribution (see www.datatables.net). You will most likely wish to modify these styles to
- * meet the layout requirements of your site.
- *
- * Common issues:
- * 'full_numbers' pagination - I use an extra selector on the body tag to ensure that there is
- * no conflict between the two pagination types. If you want to use full_numbers pagination
- * ensure that you either have "example_alt_pagination" as a body class name, or better yet,
- * modify that selector.
- * Note that the path used for Images is relative. All images are by default located in
- * ../images/ - relative to this CSS file.
- */
-
-
-/*
- * jQuery UI specific styling
- */
-
-.paging_two_button .ui-button {
- float: left;
- cursor: pointer;
- * cursor: hand;
-}
-
-.paging_full_numbers .ui-button {
- padding: 2px 6px;
- margin: 0;
- cursor: pointer;
- * cursor: hand;
-}
-
-.ui-buttonset .ui-button {
- margin-right: -0.1em !important;
-}
-
-.paging_full_numbers {
- width: 350px !important;
-}
-
-.ui-toolbar {
- padding: 5px;
-}
-
-.dataTables_paginate {
- width: auto;
-}
-
-.dataTables_info {
- padding-top: 3px;
-}
-
-table.display thead th {
- padding: 3px 0px 3px 10px;
- cursor: pointer;
- * cursor: hand;
-}
-
-div.dataTables_wrapper .ui-widget-header {
- font-weight: normal;
-}
-
-
-/*
- * Sort arrow icon positioning
- */
-table.display thead th div.DataTables_sort_wrapper {
- position: relative;
- padding-right: 20px;
- padding-right: 20px;
-}
-
-table.display thead th div.DataTables_sort_wrapper span {
- position: absolute;
- top: 50%;
- margin-top: -8px;
- right: 0;
-}
-
-
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- *
- * Everything below this line is the same as demo_table.css. This file is
- * required for 'cleanliness' of the markup
- *
- * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * DataTables features
- */
-
-.dataTables_wrapper {
- position: relative;
- min-height: 35px;
- _height: 35px;
- clear: both;
-}
-
-.dataTables_processing {
- position: absolute;
- top: 0px;
- left: 50%;
- width: 250px;
- margin-left: -125px;
- border: 1px solid #ddd;
- text-align: center;
- color: #999;
- font-size: 11px;
- padding: 2px 0;
-}
-
-.dataTables_length {
- width: 40%;
- float: left;
-}
-
-.dataTables_filter {
- width: 50%;
- float: right;
- text-align: right;
-}
-
-.dataTables_info {
- width: 50%;
- float: left;
-}
-
-.dataTables_paginate {
- float: right;
- text-align: right;
-}
-
-/* Pagination nested */
-.paginate_disabled_previous, .paginate_enabled_previous, .paginate_disabled_next, .paginate_enabled_next {
- height: 19px;
- width: 19px;
- margin-left: 3px;
- float: left;
-}
-
-.paginate_disabled_previous {
- background-image: url('../images/back_disabled.jpg');
-}
-
-.paginate_enabled_previous {
- background-image: url('../images/back_enabled.jpg');
-}
-
-.paginate_disabled_next {
- background-image: url('../images/forward_disabled.jpg');
-}
-
-.paginate_enabled_next {
- background-image: url('../images/forward_enabled.jpg');
-}
-
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * DataTables display
- */
-table.display {
- margin: 0 auto;
- width: 100%;
- clear: both;
- border-collapse: collapse;
-}
-
-table.display tfoot th {
- padding: 3px 0px 3px 10px;
- font-weight: bold;
- font-weight: normal;
-}
-
-table.display tr.heading2 td {
- border-bottom: 1px solid #aaa;
-}
-
-table.display td {
- padding: 3px 10px;
-}
-
-table.display td.center {
- text-align: center;
-}
-
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * DataTables sorting
- */
-
-.sorting_asc {
- background: url('../images/sort_asc.jpg') no-repeat center right;
-}
-
-.sorting_desc {
- background: url('../images/sort_desc.jpg') no-repeat center right;
-}
-
-.sorting {
- background: url('../images/sort_both.jpg') no-repeat center right;
-}
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * Misc
- */
-.dataTables_scroll {
- clear: both;
-}
-
-.top, .bottom {
- padding: 15px;
- background-color: #F5F5F5;
- border: 1px solid #CCCCCC;
-}
-
-.top .dataTables_info {
- float: none;
-}
-
-.clear {
- clear: both;
-}
-
-.dataTables_empty {
- text-align: center;
-}
-
-tfoot input {
- margin: 0.5em 0;
- width: 100%;
- color: #444;
-}
-
-tfoot input.search_init {
- color: #999;
-}
-
-td.group {
- background-color: #d1cfd0;
- border-bottom: 2px solid #A19B9E;
- border-top: 2px solid #A19B9E;
-}
-
-td.details {
- background-color: #d1cfd0;
- border: 2px solid #A19B9E;
-}
-
-
-.example_alt_pagination div.dataTables_info {
- width: 40%;
-}
-
-.paging_full_numbers span.paginate_button,
- .paging_full_numbers span.paginate_active {
- border: 1px solid #aaa;
- -webkit-border-radius: 5px;
- -moz-border-radius: 5px;
- padding: 2px 5px;
- margin: 0 3px;
- cursor: pointer;
- *cursor: hand;
-}
-
-.paging_full_numbers span.paginate_button {
- background-color: #ddd;
-}
-
-.paging_full_numbers span.paginate_button:hover {
- background-color: #ccc;
-}
-
-.paging_full_numbers span.paginate_active {
- background-color: #99B3FF;
-}
-
-table.display tr.even.row_selected td {
- background-color: #B0BED9;
-}
-
-table.display tr.odd.row_selected td {
- background-color: #9FAFD1;
-}
-
-/* Striping */
-tr.odd { background: rgba(255, 255, 255, 0.1); }
-tr.even { background: rgba(0, 0, 255, 0.05); }
-
-
-/*
- * Sorting classes for columns
- */
-tr.odd td.sorting_1 { background: rgba(0, 0, 0, 0.03); }
-tr.odd td.sorting_2 { background: rgba(0, 0, 0, 0.02); }
-tr.odd td.sorting_3 { background: rgba(0, 0, 0, 0.02); }
-tr.even td.sorting_1 { background: rgba(0, 0, 0, 0.08); }
-tr.even td.sorting_2 { background: rgba(0, 0, 0, 0.06); }
-tr.even td.sorting_3 { background: rgba(0, 0, 0, 0.06); }
-
-.css_left { position: relative; float: left; }
-.css_right { position: relative; float: right; }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/Sorting icons.psd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/Sorting icons.psd b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/Sorting icons.psd
deleted file mode 100644
index 53b2e06..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/Sorting icons.psd and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/back_disabled.jpg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/back_disabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/back_disabled.jpg
deleted file mode 100644
index 1e73a54..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/back_disabled.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/back_enabled.jpg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/back_enabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/back_enabled.jpg
deleted file mode 100644
index a6d764c..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/back_enabled.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/favicon.ico
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/favicon.ico b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/favicon.ico
deleted file mode 100644
index 6eeaa2a..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/favicon.ico and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/forward_disabled.jpg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/forward_disabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/forward_disabled.jpg
deleted file mode 100644
index 28a9dc5..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/forward_disabled.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/forward_enabled.jpg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/forward_enabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/forward_enabled.jpg
deleted file mode 100644
index 598c075..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/forward_enabled.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_asc.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_asc.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_asc.png
deleted file mode 100644
index a56d0e2..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_asc.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_asc_disabled.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_asc_disabled.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_asc_disabled.png
deleted file mode 100644
index b7e621e..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_asc_disabled.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_both.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_both.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_both.png
deleted file mode 100644
index 839ac4b..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_both.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_desc.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_desc.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_desc.png
deleted file mode 100644
index 90b2951..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_desc.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_desc_disabled.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_desc_disabled.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_desc_disabled.png
deleted file mode 100644
index 2409653..0000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/images/sort_desc_disabled.png and /dev/null differ
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDDS-771. ChunkGroupOutputStream
stream entries need to be properly updated on closed container exception.
Contributed by Lokesh Jain.
Posted by ae...@apache.org.
HDDS-771. ChunkGroupOutputStream stream entries need to be properly updated on closed container exception. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0ac3081
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0ac3081
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0ac3081
Branch: refs/heads/HDDS-4
Commit: e0ac3081e95bc70b13c36a2cad1565ecc35dec52
Parents: 2e8ac14
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Thu Nov 1 15:43:48 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Thu Nov 1 15:43:48 2018 +0530
----------------------------------------------------------------------
.../ozone/client/io/ChunkGroupOutputStream.java | 6 +-
.../rpc/TestCloseContainerHandlingByClient.java | 60 ++++++++++++++++++++
2 files changed, 65 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0ac3081/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index 78d69c1..3fe5d93 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -413,6 +413,11 @@ public class ChunkGroupOutputStream extends OutputStream {
return;
}
+ // update currentStreamIndex in case of closed container exception. The
+ // current stream entry cannot be used for further writes because
+ // container is closed.
+ currentStreamIndex += 1;
+
// In case where not a single chunk of data has been written to the Datanode
// yet. This block does not yet exist on the datanode but cached on the
// outputStream buffer. No need to call GetCommittedBlockLength here
@@ -429,7 +434,6 @@ public class ChunkGroupOutputStream extends OutputStream {
// allocate new block and write this data in the datanode. The cached
// data in the buffer does not exceed chunkSize.
Preconditions.checkState(buffer.position() < chunkSize);
- currentStreamIndex += 1;
// readjust the byteOffset value to the length actually been written.
byteOffset -= buffer.position();
handleWrite(buffer.array(), 0, buffer.position());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0ac3081/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index d06a0bc..c6ee872 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
@@ -287,6 +289,64 @@ public class TestCloseContainerHandlingByClient {
validateData(keyName, dataString.concat(dataString2).getBytes());
}
+ @Test
+ public void testMultiBlockWrites3() throws Exception {
+
+ String keyName = "standalone5";
+ int keyLen = 4 * blockSize;
+ OzoneOutputStream key =
+ createKey(keyName, ReplicationType.RATIS, keyLen);
+ ChunkGroupOutputStream groupOutputStream =
+ (ChunkGroupOutputStream) key.getOutputStream();
+ // With the initial size provided, it should have preallocated 4 blocks
+ Assert.assertEquals(4, groupOutputStream.getStreamEntries().size());
+ // write data 3 blocks and one more chunk
+ byte[] writtenData = fixedLengthString(keyString, keyLen).getBytes();
+ byte[] data = Arrays.copyOfRange(writtenData, 0, 3 * blockSize + chunkSize);
+ Assert.assertEquals(data.length, 3 * blockSize + chunkSize);
+ key.write(data);
+
+ Assert.assertTrue(key.getOutputStream() instanceof ChunkGroupOutputStream);
+ //get the name of a valid container
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setType(HddsProtos.ReplicationType.RATIS)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setKeyName(keyName)
+ .build();
+
+ waitForContainerClose(keyName, key,
+ HddsProtos.ReplicationType.RATIS);
+ // write 3 more chunks worth of data. It will fail and new block will be
+ // allocated. This write completes 4 blocks worth of data written to key
+ data = Arrays
+ .copyOfRange(writtenData, 3 * blockSize + chunkSize, keyLen);
+ key.write(data);
+
+ key.close();
+ // read the key from OM again and match the length and data.
+ OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
+ List<OmKeyLocationInfo> keyLocationInfos =
+ keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
+ OzoneVolume volume = objectStore.getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ OzoneInputStream inputStream = bucket.readKey(keyName);
+ byte[] readData = new byte[keyLen];
+ inputStream.read(readData);
+ Assert.assertArrayEquals(writtenData, readData);
+
+ // Though we have written only block initially, the close will hit
+ // closeContainerException and remaining data in the chunkOutputStream
+ // buffer will be copied into a different allocated block and will be
+ // committed.
+ Assert.assertEquals(5, keyLocationInfos.size());
+ Assert.assertEquals(4 * blockSize, keyInfo.getDataSize());
+ long length = 0;
+ for (OmKeyLocationInfo locationInfo : keyLocationInfos) {
+ length += locationInfo.getLength();
+ }
+ Assert.assertEquals(4 * blockSize, length);
+ }
+
private void waitForContainerClose(String keyName,
OzoneOutputStream outputStream, HddsProtos.ReplicationType type)
throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HDDS-773. Loading ozone s3 bucket
browser could be failed. Contributed by Elek Marton.
Posted by ae...@apache.org.
HDDS-773. Loading ozone s3 bucket browser could be failed. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/478b2cba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/478b2cba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/478b2cba
Branch: refs/heads/HDDS-4
Commit: 478b2cba0de5aadf655ac0b5a607760d46cc2a1e
Parents: b519f3f
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Wed Oct 31 07:54:23 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Wed Oct 31 07:54:23 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/dist/src/main/smoketest/s3/README.md | 2 +-
hadoop-ozone/s3gateway/pom.xml | 6 ++++++
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 13 ++++++++-----
3 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/478b2cba/hadoop-ozone/dist/src/main/smoketest/s3/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/README.md b/hadoop-ozone/dist/src/main/smoketest/s3/README.md
index 884ba2e..70ccda7 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/README.md
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/README.md
@@ -23,5 +23,5 @@ You need to
3. Set bucket/endpointurl during the robot test execution
```
-robot -v bucket:ozonetest -v OZONE_S3_SET_CREDENTIALS:false -v ENDPOINT_URL:https://s3.us-east-2.amazonaws.com smoketest/s3
+robot -v bucket:ozonetest -v OZONE_TEST:false -v OZONE_S3_SET_CREDENTIALS:false -v ENDPOINT_URL:https://s3.us-east-2.amazonaws.com smoketest/s3
```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/478b2cba/hadoop-ozone/s3gateway/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml
index 06012cf..52eee5d 100644
--- a/hadoop-ozone/s3gateway/pom.xml
+++ b/hadoop-ozone/s3gateway/pom.xml
@@ -174,5 +174,11 @@
<version>2.15.0</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>findbugs</artifactId>
+ <version>3.0.1</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/478b2cba/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 04e2348..bfbbb33 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteResponse.Error;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.ozone.s3.util.S3utils;
import org.apache.http.HttpStatus;
@@ -70,6 +71,7 @@ public class BucketEndpoint extends EndpointBase {
* for more details.
*/
@GET
+ @SuppressFBWarnings
public Response list(
@PathParam("bucket") String bucketName,
@QueryParam("delimiter") String delimiter,
@@ -83,12 +85,12 @@ public class BucketEndpoint extends EndpointBase {
@Context HttpHeaders hh) throws OS3Exception, IOException {
if (browser != null) {
- try (InputStream browserPage = getClass()
- .getResourceAsStream("/browser.html")) {
- return Response.ok(browserPage,
+ InputStream browserPage = getClass()
+ .getResourceAsStream("/browser.html");
+ return Response.ok(browserPage,
MediaType.TEXT_HTML_TYPE)
.build();
- }
+
}
if (prefix == null) {
@@ -295,7 +297,8 @@ public class BucketEndpoint extends EndpointBase {
keyMetadata.setSize(next.getDataSize());
keyMetadata.setETag("" + next.getModificationTime());
keyMetadata.setStorageClass("STANDARD");
- keyMetadata.setLastModified(Instant.ofEpochMilli(next.getModificationTime()));
+ keyMetadata.setLastModified(Instant.ofEpochMilli(
+ next.getModificationTime()));
response.addKey(keyMetadata);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDFS-14008. NN should log
snapshotdiff report. Contributed by Pranay Singh.
Posted by ae...@apache.org.
HDFS-14008. NN should log snapshotdiff report. Contributed by Pranay Singh.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d98b881e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d98b881e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d98b881e
Branch: refs/heads/HDDS-4
Commit: d98b881e9ab826eb7b70485d0de2a41ab7345334
Parents: 2ab611d
Author: Pranay Singh <ps...@cloudera.com>
Authored: Thu Nov 1 17:25:11 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Nov 1 17:26:00 2018 -0700
----------------------------------------------------------------------
.../hdfs/protocol/SnapshotDiffReport.java | 65 ++++++++++++++++++++
.../hdfs/server/namenode/FSNamesystem.java | 20 ++++++
.../snapshot/DirectorySnapshottableFeature.java | 5 ++
.../namenode/snapshot/SnapshotDiffInfo.java | 50 ++++++++++++++-
4 files changed, 139 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
index 8ee4ec7..7bc95c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
@@ -170,14 +170,75 @@ public class SnapshotDiffReport {
/** end point of the diff */
private final String toSnapshot;
+
/** list of diff */
private final List<DiffReportEntry> diffList;
+ /**
+ * Records the stats related to Snapshot diff operation.
+ */
+ public static class DiffStats {
+ // Total dirs processed
+ private long totalDirsProcessed;
+
+ // Total dirs compared
+ private long totalDirsCompared;
+
+ // Total files processed
+ private long totalFilesProcessed;
+
+ // Total files compared
+ private long totalFilesCompared;
+
+ // Total children listing time
+ private final long totalChildrenListingTime;
+
+ public DiffStats(long totalDirsProcessed, long totalDirsCompared,
+ long totalFilesProcessed, long totalFilesCompared,
+ long totalChildrenListingTime) {
+ this.totalDirsCompared = totalDirsProcessed;
+ this.totalDirsProcessed = totalDirsCompared;
+ this.totalFilesCompared = totalFilesProcessed;
+ this.totalFilesProcessed = totalFilesCompared;
+ this.totalChildrenListingTime = totalChildrenListingTime;
+ }
+
+ public long getTotalDirsProcessed() {
+ return this.totalDirsProcessed;
+ }
+
+ public long getTotalDirsCompared() {
+ return this.totalDirsCompared;
+ }
+
+ public long getTotalFilesProcessed() {
+ return this.totalFilesProcessed;
+ }
+
+ public long getTotalFilesCompared() {
+ return this.totalFilesCompared;
+ }
+
+ public long getTotalChildrenListingTime() {
+ return totalChildrenListingTime;
+ }
+ }
+
+ /* Stats associated with the SnapshotDiff Report. */
+ private final DiffStats diffStats;
+
public SnapshotDiffReport(String snapshotRoot, String fromSnapshot,
String toSnapshot, List<DiffReportEntry> entryList) {
+ this(snapshotRoot, fromSnapshot, toSnapshot, new DiffStats(0, 0, 0, 0, 0),
+ entryList);
+ }
+
+ public SnapshotDiffReport(String snapshotRoot, String fromSnapshot,
+ String toSnapshot, DiffStats dStat, List<DiffReportEntry> entryList) {
this.snapshotRoot = snapshotRoot;
this.fromSnapshot = fromSnapshot;
this.toSnapshot = toSnapshot;
+ this.diffStats = dStat;
this.diffList = entryList != null ? entryList : Collections
.<DiffReportEntry> emptyList();
}
@@ -197,6 +258,10 @@ public class SnapshotDiffReport {
return toSnapshot;
}
+ public DiffStats getStats() {
+ return this.diffStats;
+ }
+
/** @return {@link #diffList} */
public List<DiffReportEntry> getDiffList() {
return diffList;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index d1904fa..8b1cdf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.server.namenode.metrics.ReplicatedBlocksMBean;
import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
+import org.apache.hadoop.util.Time;
import static org.apache.hadoop.util.Time.now;
import static org.apache.hadoop.util.Time.monotonicNow;
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
@@ -6624,6 +6625,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
SnapshotDiffReport getSnapshotDiffReport(String path,
String fromSnapshot, String toSnapshot) throws IOException {
+ long begTime = Time.monotonicNow();
final String operationName = "computeSnapshotDiff";
SnapshotDiffReport diffs = null;
checkOperation(OperationCategory.READ);
@@ -6634,6 +6636,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
path : Snapshot.getSnapshotPath(path, toSnapshot);
final FSPermissionChecker pc = getPermissionChecker();
readLock();
+ long actualTime = Time.monotonicNow();
try {
checkOperation(OperationCategory.READ);
diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, pc, snapshotManager,
@@ -6646,6 +6649,23 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
} finally {
readUnlock(operationName);
}
+
+ if (diffs != null) {
+ SnapshotDiffReport.DiffStats dstat = diffs.getStats();
+ LOG.info("SnapshotDiffReport '"
+ + fromSnapshot
+ + "' to '" + toSnapshot + "'. Total comparison dirs: "
+ + dstat.getTotalDirsCompared()
+ + "/" + dstat.getTotalDirsProcessed()
+ + ", files: "
+ + dstat.getTotalFilesCompared()
+ + "/" + dstat.getTotalFilesProcessed()
+ + ". Time snapChildrenListing: "
+ + dstat.getTotalChildrenListingTime() / 1000.0 + "s, actual: "
+ + ((Time.monotonicNow() - actualTime) / 1000.0) + "s, total: "
+ + ((Time.monotonicNow() - begTime) / 1000.0) + "s.");
+ }
+
logAuditEvent(success, operationName, fromSnapshotRoot,
toSnapshotRoot, null);
return diffs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
index 15aa22a..073b88b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
@@ -389,9 +389,13 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
if (change) {
diffReport.addDirDiff(dir, relativePath, diff);
}
+ } else {
+ diffReport.incrementDirsProcessed();
}
+ long startTime = Time.monotonicNow();
ReadOnlyList<INode> children = dir.getChildrenList(earlierSnapshot
.getId());
+ diffReport.addChildrenListingTime(Time.monotonicNow() - startTime);
for (INode child : children) {
final byte[] name = child.getLocalNameBytes();
boolean toProcess = !diff.containsDeleted(name);
@@ -418,6 +422,7 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
if (change) {
diffReport.addFileDiff(file, relativePath);
}
+ diffReport.incrementFilesProcessed();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d98b881e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
index 4c74afd..ab6f415 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
@@ -125,6 +125,21 @@ class SnapshotDiffInfo {
private final Map<Long, RenameEntry> renameMap =
new HashMap<Long, RenameEntry>();
+ // Total directories compared
+ private long totalDirsCompared;
+
+ // Total directories
+ private long totalDirsProcessed;
+
+ // Total files compared
+ private long totalFilesCompared;
+
+ // Total files
+ private long totalFilesProcessed;
+
+ // Total children listing time
+ private long childrenListingTime;
+
SnapshotDiffInfo(INodeDirectory snapshotRootDir,
INodeDirectory snapshotDiffScopeDir, Snapshot start, Snapshot end) {
Preconditions.checkArgument(snapshotRootDir.isSnapshottable() &&
@@ -133,6 +148,10 @@ class SnapshotDiffInfo {
this.snapshotDiffScopeDir = snapshotDiffScopeDir;
this.from = start;
this.to = end;
+ this.totalDirsCompared = 0;
+ this.totalDirsProcessed = 0;
+ this.totalFilesCompared = 0;
+ this.totalFilesProcessed = 0;
}
/** Add a dir-diff pair */
@@ -164,6 +183,29 @@ class SnapshotDiffInfo {
return to;
}
+
+ void incrementDirsCompared() {
+ this.totalDirsCompared++;
+ incrementDirsProcessed();
+ }
+
+ void incrementDirsProcessed() {
+ this.totalDirsProcessed++;
+ }
+
+ void incrementFilesCompared() {
+ this.totalFilesCompared++;
+ incrementFilesProcessed();
+ }
+
+ void incrementFilesProcessed() {
+ this.totalFilesProcessed++;
+ }
+
+ public void addChildrenListingTime(long millis) {
+ this.childrenListingTime += millis;
+ }
+
private RenameEntry getEntry(long inodeId) {
RenameEntry entry = renameMap.get(inodeId);
if (entry == null) {
@@ -203,9 +245,15 @@ class SnapshotDiffInfo {
diffReportList.addAll(subList);
}
}
+
+ SnapshotDiffReport.DiffStats dStats = new SnapshotDiffReport.DiffStats(
+ this.totalDirsCompared, this.totalDirsProcessed,
+ this.totalFilesCompared, this.totalFilesProcessed,
+ this.childrenListingTime);
+
return new SnapshotDiffReport(snapshotRoot.getFullPathName(),
Snapshot.getSnapshotName(from), Snapshot.getSnapshotName(to),
- diffReportList);
+ dStats, diffReportList);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDDS-659. Implement pagination in
GET bucket (object list) endpoint. Contributed by Bharat Viswanadham.
Posted by ae...@apache.org.
HDDS-659. Implement pagination in GET bucket (object list) endpoint. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b519f3f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b519f3f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b519f3f2
Branch: refs/heads/HDDS-4
Commit: b519f3f2a0ae960391ce7bff59f1fdd21a22e030
Parents: ecac351
Author: Márton Elek <el...@apache.org>
Authored: Wed Oct 31 12:21:38 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Wed Oct 31 13:29:01 2018 +0100
----------------------------------------------------------------------
.../ozone/s3/endpoint/BucketEndpoint.java | 109 ++++++---
.../ozone/s3/endpoint/ListObjectResponse.java | 22 ++
.../apache/hadoop/ozone/s3/util/S3Consts.java | 1 +
.../apache/hadoop/ozone/s3/util/S3utils.java | 73 ++++++
.../hadoop/ozone/client/OzoneBucketStub.java | 7 +-
.../hadoop/ozone/s3/endpoint/TestBucketGet.java | 227 ++++++++++++++++++-
6 files changed, 400 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 8f554ed..04e2348 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -36,7 +36,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
import java.util.Iterator;
-import javax.ws.rs.core.Response.ResponseBuilder;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneKey;
@@ -48,10 +47,13 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.s3.util.S3utils;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
+
/**
* Bucket level rest endpoints.
*/
@@ -76,6 +78,8 @@ public class BucketEndpoint extends EndpointBase {
@DefaultValue("1000") @QueryParam("max-keys") int maxKeys,
@QueryParam("prefix") String prefix,
@QueryParam("browser") String browser,
+ @QueryParam("continuation-token") String continueToken,
+ @QueryParam("start-after") String startAfter,
@Context HttpHeaders hh) throws OS3Exception, IOException {
if (browser != null) {
@@ -87,60 +91,91 @@ public class BucketEndpoint extends EndpointBase {
}
}
- if (delimiter == null) {
- delimiter = "/";
- }
if (prefix == null) {
prefix = "";
}
OzoneBucket bucket = getBucket(bucketName);
- Iterator<? extends OzoneKey> ozoneKeyIterator = bucket.listKeys(prefix);
+ Iterator<? extends OzoneKey> ozoneKeyIterator;
+
+ String decodedToken = S3utils.decodeContinueToken(continueToken);
+
+ if (startAfter != null && continueToken != null) {
+ // If continuation token and start after both are provided, then we
+ // ignore start After
+ ozoneKeyIterator = bucket.listKeys(prefix, decodedToken);
+ } else if (startAfter != null && continueToken == null) {
+ ozoneKeyIterator = bucket.listKeys(prefix, startAfter);
+ } else if (startAfter == null && continueToken != null){
+ ozoneKeyIterator = bucket.listKeys(prefix, decodedToken);
+ } else {
+ ozoneKeyIterator = bucket.listKeys(prefix);
+ }
+
ListObjectResponse response = new ListObjectResponse();
response.setDelimiter(delimiter);
response.setName(bucketName);
response.setPrefix(prefix);
response.setMarker("");
- response.setMaxKeys(1000);
- response.setEncodingType("url");
+ response.setMaxKeys(maxKeys);
+ response.setEncodingType(ENCODING_TYPE);
response.setTruncated(false);
+ response.setContinueToken(continueToken);
String prevDir = null;
+ String lastKey = null;
+ int count = 0;
while (ozoneKeyIterator.hasNext()) {
OzoneKey next = ozoneKeyIterator.next();
String relativeKeyName = next.getName().substring(prefix.length());
- int depth =
- StringUtils.countMatches(relativeKeyName, delimiter);
+ int depth = StringUtils.countMatches(relativeKeyName, delimiter);
+ if (delimiter != null) {
+ if (depth > 0) {
+ // means key has multiple delimiters in its value.
+ // ex: dir/dir1/dir2, where delimiter is "/" and prefix is dir/
+ String dirName = relativeKeyName.substring(0, relativeKeyName
+ .indexOf(delimiter));
+ if (!dirName.equals(prevDir)) {
+ response.addPrefix(prefix + dirName + delimiter);
+ prevDir = dirName;
+ count++;
+ }
+ } else if (relativeKeyName.endsWith(delimiter)) {
+ // means or key is same as prefix with delimiter at end and ends with
+ // delimiter. ex: dir/, where prefix is dir and delimiter is /
+ response.addPrefix(relativeKeyName);
+ count++;
+ } else {
+ // means our key is matched with prefix if prefix is given and it
+ // does not have any common prefix.
+ addKey(response, next);
+ count++;
+ }
+ } else {
+ addKey(response, next);
+ count++;
+ }
- if (prefix.length() > 0 && !prefix.endsWith(delimiter)
- && relativeKeyName.length() > 0) {
- response.addPrefix(prefix + "/");
+ if (count == maxKeys) {
+ lastKey = next.getName();
break;
}
- if (depth > 0) {
- String dirName = relativeKeyName
- .substring(0, relativeKeyName.indexOf(delimiter));
- if (!dirName.equals(prevDir)) {
- response.addPrefix(
- prefix + dirName + delimiter);
- prevDir = dirName;
- }
- } else if (relativeKeyName.endsWith(delimiter)) {
- response.addPrefix(relativeKeyName);
- } else if (relativeKeyName.length() > 0) {
- KeyMetadata keyMetadata = new KeyMetadata();
- keyMetadata.setKey(next.getName());
- keyMetadata.setSize(next.getDataSize());
- keyMetadata.setETag("" + next.getModificationTime());
- keyMetadata.setStorageClass("STANDARD");
- keyMetadata
- .setLastModified(Instant.ofEpochMilli(next.getModificationTime()));
- response.addKey(keyMetadata);
- }
}
+
+ response.setKeyCount(count);
+
+ if (count < maxKeys) {
+ response.setTruncated(false);
+ } else if(ozoneKeyIterator.hasNext()) {
+ response.setTruncated(true);
+ response.setNextToken(S3utils.generateContinueToken(lastKey));
+ } else {
+ response.setTruncated(false);
+ }
+
response.setKeyCount(
response.getCommonPrefixes().size() + response.getContents().size());
return Response.ok(response).build();
@@ -253,4 +288,14 @@ public class BucketEndpoint extends EndpointBase {
}
return result;
}
+
+ private void addKey(ListObjectResponse response, OzoneKey next) {
+ KeyMetadata keyMetadata = new KeyMetadata();
+ keyMetadata.setKey(next.getName());
+ keyMetadata.setSize(next.getDataSize());
+ keyMetadata.setETag("" + next.getModificationTime());
+ keyMetadata.setStorageClass("STANDARD");
+ keyMetadata.setLastModified(Instant.ofEpochMilli(next.getModificationTime()));
+ response.addKey(keyMetadata);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
index b9ab977..adb5f20 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListObjectResponse.java
@@ -60,6 +60,12 @@ public class ListObjectResponse {
@XmlElement(name = "IsTruncated")
private boolean isTruncated;
+ @XmlElement(name = "NextContinuationToken")
+ private String nextToken;
+
+ @XmlElement(name = "continueToken")
+ private String continueToken;
+
@XmlElement(name = "Contents")
private List<KeyMetadata> contents = new ArrayList<>();
@@ -148,6 +154,22 @@ public class ListObjectResponse {
commonPrefixes.add(new CommonPrefix(relativeKeyName));
}
+ public String getNextToken() {
+ return nextToken;
+ }
+
+ public void setNextToken(String nextToken) {
+ this.nextToken = nextToken;
+ }
+
+ public String getContinueToken() {
+ return continueToken;
+ }
+
+ public void setContinueToken(String continueToken) {
+ this.continueToken = continueToken;
+ }
+
public int getKeyCount() {
return keyCount;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 2e7b965..70d8a96 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -15,5 +15,6 @@ public final class S3Consts {
public static final String COPY_SOURCE_HEADER = "x-amz-copy-source";
public static final String STORAGE_CLASS_HEADER = "x-amz-storage-class";
+ public static final String ENCODING_TYPE = "url";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
new file mode 100644
index 0000000..8af0927
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
@@ -0,0 +1,73 @@
+package org.apache.hadoop.ozone.s3.util;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Utility class for S3.
+ */
+public final class S3utils {
+
+ private S3utils() {
+
+ }
+ private static final String CONTINUE_TOKEN_SEPERATOR = "-";
+
+ /**
+ * Generate a continuation token which is used in get Bucket.
+ * @param key
+ * @return if key is not null return continuation token, else returns null.
+ */
+ public static String generateContinueToken(String key) {
+ if (key != null) {
+ byte[] byteData = key.getBytes(StandardCharsets.UTF_8);
+ String hex = Hex.encodeHexString(byteData);
+ String digest = DigestUtils.sha256Hex(key);
+ return hex + CONTINUE_TOKEN_SEPERATOR + digest;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Decode a continuation token which is used in get Bucket.
+ * @param key
+ * @return if key is not null return decoded token, otherwise returns null.
+ * @throws OS3Exception
+ */
+ public static String decodeContinueToken(String key) throws OS3Exception {
+ if (key != null) {
+ int indexSeparator = key.indexOf(CONTINUE_TOKEN_SEPERATOR);
+ if (indexSeparator == -1) {
+ throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, key);
+ }
+ String hex = key.substring(0, indexSeparator);
+ String digest = key.substring(indexSeparator + 1);
+ try {
+ byte[] actualKeyBytes = Hex.decodeHex(hex);
+ String digestActualKey = DigestUtils.sha256Hex(actualKeyBytes);
+ if (digest.equals(digestActualKey)) {
+ return new String(actualKeyBytes);
+ } else {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
+ .INVALID_ARGUMENT, key);
+ ex.setErrorMessage("The continuation token provided is incorrect");
+ throw ex;
+ }
+ } catch (DecoderException ex) {
+ OS3Exception os3Exception = S3ErrorTable.newError(S3ErrorTable
+ .INVALID_ARGUMENT, key);
+ os3Exception.setErrorMessage("The continuation token provided is " +
+ "incorrect");
+ throw os3Exception;
+ }
+ } else {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 6aff087..918e9d0 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -27,6 +27,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.client.ReplicationFactor;
@@ -116,7 +117,8 @@ public class OzoneBucketStub extends OzoneBucket {
@Override
public Iterator<? extends OzoneKey> listKeys(String keyPrefix) {
- return keyDetails.values()
+ Map<String, OzoneKey> sortedKey = new TreeMap<String, OzoneKey>(keyDetails);
+ return sortedKey.values()
.stream()
.filter(key -> key.getName().startsWith(keyPrefix))
.collect(Collectors.toList())
@@ -126,7 +128,8 @@ public class OzoneBucketStub extends OzoneBucket {
@Override
public Iterator<? extends OzoneKey> listKeys(String keyPrefix,
String prevKey) {
- return keyDetails.values()
+ Map<String, OzoneKey> sortedKey = new TreeMap<String, OzoneKey>(keyDetails);
+ return sortedKey.values()
.stream()
.filter(key -> key.getName().compareTo(prevKey) > 0)
.filter(key -> key.getName().startsWith(keyPrefix))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b519f3f2/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
index 54534ed..8b70d29 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.junit.Assert;
import org.junit.Test;
+import static org.junit.Assert.fail;
+
/**
* Testing basic object list browsing.
*/
@@ -45,7 +47,7 @@ public class TestBucketGet {
ListObjectResponse getBucketResponse =
(ListObjectResponse) getBucket
- .list("b1", "/", null, null, 100, "", null, null)
+ .list("b1", "/", null, null, 100, "", null, null, null, null)
.getEntity();
Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
@@ -68,8 +70,8 @@ public class TestBucketGet {
getBucket.setClient(client);
ListObjectResponse getBucketResponse =
- (ListObjectResponse) getBucket
- .list("b1", "/", null, null, 100, "dir1", null, null).getEntity();
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 100,
+ "dir1", null, null, null, null).getEntity();
Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
Assert.assertEquals("dir1/",
@@ -85,13 +87,16 @@ public class TestBucketGet {
BucketEndpoint getBucket = new BucketEndpoint();
OzoneClient ozoneClient =
- createClientWithKeys("dir1/file2", "dir1/dir2/file2");
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2");
getBucket.setClient(ozoneClient);
ListObjectResponse getBucketResponse =
(ListObjectResponse) getBucket
- .list("b1", "/", null, null, 100, "dir1/", null, null).getEntity();
+ .list("b1", "/", null, null, 100, "dir1/", null, null,
+ null, null)
+ .getEntity();
Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size());
Assert.assertEquals("dir1/dir2/",
@@ -103,6 +108,218 @@ public class TestBucketGet {
}
+
+ @Test
+ public void listWithPrefixAndDelimiter() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "file2");
+
+ getBucket.setClient(ozoneClient);
+
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 100,
+ "dir1", null, null, null, null).getEntity();
+
+ Assert.assertEquals(3, getBucketResponse.getCommonPrefixes().size());
+
+ }
+
+ @Test
+ public void listWithPrefixAndDelimiter1() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "file2");
+
+ getBucket.setClient(ozoneClient);
+
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 100,
+ "", null, null, null, null).getEntity();
+
+ Assert.assertEquals(3, getBucketResponse.getCommonPrefixes().size());
+ Assert.assertEquals("file2", getBucketResponse.getContents().get(0)
+ .getKey());
+
+ }
+
+ @Test
+ public void listWithPrefixAndDelimiter2() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "file2");
+
+ getBucket.setClient(ozoneClient);
+
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 100,
+ "dir1bh", null, null, "dir1/dir2/file2", null).getEntity();
+
+ Assert.assertEquals(2, getBucketResponse.getCommonPrefixes().size());
+
+ }
+
+ @Test
+ public void listWithContinuationToken() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "file2");
+
+ getBucket.setClient(ozoneClient);
+
+ int maxKeys = 2;
+ // As we have 5 keys, with max keys 2 we should call list 3 times.
+
+ // First time
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys,
+ "", null, null, null, null).getEntity();
+
+ Assert.assertTrue(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 2);
+
+ // 2nd time
+ String continueToken = getBucketResponse.getNextToken();
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys,
+ "", null, continueToken, null, null).getEntity();
+ Assert.assertTrue(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 2);
+
+
+ continueToken = getBucketResponse.getNextToken();
+
+ //3rd time
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys,
+ "", null, continueToken, null, null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 1);
+
+ }
+
+ @Test
+ /**
+ * This test is with prefix and delimiter and verify continuation-token
+ * behavior.
+ */
+ public void listWithContinuationToken1() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file1", "dir1bh/file1",
+ "dir1bha/file1", "dir0/file1", "dir2/file1");
+
+ getBucket.setClient(ozoneClient);
+
+ int maxKeys = 2;
+ // As we have 5 keys, with max keys 2 we should call list 3 times.
+
+ // First time
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys,
+ "dir", null, null, null, null).getEntity();
+
+ Assert.assertTrue(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getCommonPrefixes().size() == 2);
+
+
+ // 2nd time
+ String continueToken = getBucketResponse.getNextToken();
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys,
+ "dir", null, continueToken, null, null).getEntity();
+ Assert.assertTrue(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getCommonPrefixes().size() == 2);
+
+
+ //3rd time
+ continueToken = getBucketResponse.getNextToken();
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys,
+ "dir", null, continueToken, null, null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getCommonPrefixes().size() == 1);
+
+ }
+
+ @Test
+ public void listWithContinuationTokenFail() throws OS3Exception, IOException {
+
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file2", "dir1/dir2/file2", "dir1bh/file",
+ "dir1bha/file2", "dir1", "dir2", "dir3");
+
+ getBucket.setClient(ozoneClient);
+
+ try {
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", "/", null, null, 2,
+ "dir", null, "random", null, null).getEntity();
+ fail("listWithContinuationTokenFail");
+ } catch (OS3Exception ex) {
+ Assert.assertEquals("random", ex.getResource());
+ Assert.assertEquals("Invalid Argument", ex.getErrorMessage());
+ }
+
+ }
+
+
+ @Test
+ public void testStartAfter() throws IOException, OS3Exception {
+ BucketEndpoint getBucket = new BucketEndpoint();
+
+ OzoneClient ozoneClient =
+ createClientWithKeys("dir1/file1", "dir1bh/file1",
+ "dir1bha/file1", "dir0/file1", "dir2/file1");
+
+ getBucket.setClient(ozoneClient);
+
+ ListObjectResponse getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null, 1000,
+ null, null, null, null, null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 5);
+
+ //As our list output is sorted, after seeking to startAfter, we shall
+ // have 4 keys.
+ String startAfter = "dir0/file1";
+
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null,
+ 1000, null, null, null, startAfter, null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 4);
+
+ getBucketResponse =
+ (ListObjectResponse) getBucket.list("b1", null, null, null,
+ 1000, null, null, null, "random", null).getEntity();
+
+ Assert.assertFalse(getBucketResponse.isTruncated());
+ Assert.assertTrue(getBucketResponse.getContents().size() == 0);
+
+
+ }
+
private OzoneClient createClientWithKeys(String... keys) throws IOException {
OzoneClient client = new OzoneClientStub();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDFS-14033. [libhdfs++] Disable
libhdfs++ build on systems that do not support thread_local. Contributed by
Anatoli Shein.
Posted by ae...@apache.org.
HDFS-14033. [libhdfs++] Disable libhdfs++ build on systems that do not support thread_local. Contributed by Anatoli Shein.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c438abe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c438abe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c438abe
Branch: refs/heads/HDDS-4
Commit: 9c438abe52d4ee0b25345a4b7ec1697dd66f85e9
Parents: fac9f91
Author: Sunil G <su...@apache.org>
Authored: Wed Oct 31 12:32:49 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Oct 31 12:32:49 2018 +0530
----------------------------------------------------------------------
.../src/CMakeLists.txt | 22 +++++++++++++++++++-
.../src/main/native/libhdfspp/CMakeLists.txt | 4 ++--
2 files changed, 23 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c438abe/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
index 1813ec1..026be9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
@@ -138,7 +138,27 @@ endif()
add_subdirectory(main/native/libhdfs)
add_subdirectory(main/native/libhdfs-tests)
-add_subdirectory(main/native/libhdfspp)
+
+# Temporary fix to disable Libhdfs++ build on older systems that do not support thread_local
+include(CheckCXXSourceCompiles)
+unset (THREAD_LOCAL_SUPPORTED CACHE)
+set (CMAKE_REQUIRED_DEFINITIONS "-std=c++11")
+set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+check_cxx_source_compiles(
+ "#include <thread>
+ int main(void) {
+ thread_local int s;
+ return 0;
+ }"
+ THREAD_LOCAL_SUPPORTED)
+if (THREAD_LOCAL_SUPPORTED)
+ add_subdirectory(main/native/libhdfspp)
+else()
+ message(WARNING
+ "WARNING: Libhdfs++ library was not built because the required feature thread_local storage \
+ is not supported by your compiler. Known compilers that support this feature: GCC 4.8+, Visual Studio 2015+, \
+ Clang (community version 3.3+), Clang (version for Xcode 8+ and iOS 9+).")
+endif (THREAD_LOCAL_SUPPORTED)
if(REQUIRE_LIBWEBHDFS)
add_subdirectory(contrib/libwebhdfs)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c438abe/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index 63fa80d..411320a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -63,8 +63,8 @@ check_cxx_source_compiles(
if (NOT THREAD_LOCAL_SUPPORTED)
message(FATAL_ERROR
"FATAL ERROR: The required feature thread_local storage is not supported by your compiler. \
- Known compilers that support this feature: GCC, Visual Studio, Clang (community version), \
- Clang (version for iOS 9 and later).")
+ Known compilers that support this feature: GCC 4.8+, Visual Studio 2015+, Clang (community \
+ version 3.3+), Clang (version for Xcode 8+ and iOS 9+).")
endif (NOT THREAD_LOCAL_SUPPORTED)
# Check if PROTOC library was compiled with the compatible compiler by trying
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HADOOP-15855. Review hadoop
credential doc, including object store details. Contributed by Steve Loughran.
Posted by ae...@apache.org.
HADOOP-15855. Review hadoop credential doc, including object store details.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62d98ca9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62d98ca9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62d98ca9
Branch: refs/heads/HDDS-4
Commit: 62d98ca92aee15d1790d169bfdf0043b05b748ce
Parents: 7757331
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 30 15:58:04 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 30 15:58:04 2018 +0000
----------------------------------------------------------------------
.../src/site/markdown/CredentialProviderAPI.md | 130 ++++++++++++++-----
.../hadoop/crypto/key/TestKeyProvider.java | 32 +++--
2 files changed, 119 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d98ca9/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
index bd1c2c7..0c5f486 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
@@ -32,10 +32,12 @@ Overview
Usage
-----
+
### Usage Overview
Let's provide a quick overview of the use of the credential provider framework for protecting passwords or other sensitive tokens in hadoop.
##### Why is it used?
+
There are certain deployments that are very sensitive to how sensitive tokens like passwords are stored and managed within the cluster. For instance, there may be security best practices and policies in place that require such things to never be stored in clear text, for example. Enterprise deployments may be required to use a preferred solution for managing credentials and we need a way to plug in integrations for them.
##### General Usage Pattern
@@ -48,46 +50,46 @@ There are numerous places within the Hadoop project and ecosystem that can lever
3. Features or components that leverage the new [Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) method to resolve their credentials will automatically pick up support for the credential provider API.
- By using the same property names as are used for existing clear text passwords, this mechanism allows for the migration to credential providers while providing backward compatibility for clear text.
- The entire credential provider path is interrogated before falling back to clear text passwords in config.
-4. Features or components that do not use the hadoop Configuration class for config or have other internal uses for the credential providers may choose to write to the CredentialProvider API itself. An example of its use will be included in this document but may also be found within [Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) and within the unit tests of features that have added support and need to provision credentials for the tests.
+4. Features or components that do not use Hadoop's `org.apache.hadoop.conf.Configuration` class for configuration or have other internal uses for the credential providers may choose to use the `CredentialProvider` API itself. An example of its use can be found within [Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) and within its unit tests.
##### Provision Credentials
-Example: ssl.server.keystore.password
+Example: `ssl.server.keystore.password`
-```
- hadoop credential create ssl.server.keystore.password -value 123
- -provider localjceks://file/home/lmccay/aws.jceks
+```bash
+hadoop credential create ssl.server.keystore.password -value 123 \
+ -provider localjceks://file/home/lmccay/aws.jceks
```
-Note that the alias names are the same as the configuration properties that were used to get the
-credentials from the Configuration.get method. Reusing these names allows for intuitive
-migration to the use of credential providers and fall back logic for backward compatibility.
+The alias names are the same as the configuration properties that were used to get the
+credentials from the `Configuration.get()` methods.
##### Configuring the Provider Path
+
Now, we need to make sure that this provisioned credential store is known at runtime by the
[Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) method. If there is no credential provider path configuration then
-getPassword will skip the credential provider API interrogation. So, it is important that the
-following be configured within core-site.xml or your component's equivalent.
-
-```
- <property>
- <name>hadoop.security.credential.provider.path</name>
- <value>localjceks://file/home/lmccay/aws.jceks</value>
- <description>Path to interrogate for protected credentials.</description>
- </property>
+`Configuration.getPassword()` will skip the credential provider API interrogation. So, it is important that the
+following be configured within `core-site.xml` or your component's equivalent.
+
+```xml
+<property>
+ <name>hadoop.security.credential.provider.path</name>
+ <value>localjceks://file/home/lmccay/aws.jceks</value>
+ <description>Path to interrogate for protected credentials.</description>
+</property>
```
A couple additional things to note about the provider path:
1. The scheme is used to indicate the type of provider in the above case the
- localjceks provider does not have a dependency on the Hadoop fs abstraction
+ `localjceks` provider does not have a dependency on the Hadoop FileSystem APIs.
and is needed sometimes to avoid a recursive dependency. Another provider
- represented by jceks, does use the Hadoop fs abstraction and therefore has
- support for keystores provisioned within HDFS. A third provider type is the
- user type. This provider can manage credentials stored within the Credentials
+ represented by `jceks`, does use the Hadoop FileSystem APIs and can
+ support keystores provisioned within HDFS or other compatible filesystems.
+ A third provider type is the `user` type.
+ This provider can manage credentials stored within the Credentials
file for a process.
2. The path configuration accepts a comma separated path of providers or
- credential stores. The [Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) method will walk through
- all of the providers until it resolves the alias or exhausts the list.
+ credential stores. The [Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) method will query each of the providers, in order until it resolves the alias or exhausts the list.
Depending on the runtime needs for credentials, we may need to configure
a chain of providers to check.
@@ -98,13 +100,13 @@ In summary, first, provision the credentials into a provider then configure the
|:---- |:---- |:---|
|LDAPGroupsMapping |LDAPGroupsMapping is used to look up the groups for a given user in LDAP. The CredentialProvider API is used to protect the LDAP bind password and those needed for SSL.|[LDAP Groups Mapping](GroupsMapping.html#LDAP_Groups_Mapping)|
|SSL Passwords |FileBasedKeyStoresFactory leverages the credential provider API in order to resolve the SSL related passwords.|TODO|
-|HDFS |DFSUtil leverages Configuration.getPassword method to use the credential provider API and/or fallback to the clear text value stored in ssl-server.xml. Zookeeper based federation state store and failover controller use Configuration.getPassword to get the Zookeeper authentication info, with fallback provided to clear text auth info.|TODO|
+|HDFS |DFSUtil uses `Configuration.getPassword()` use the credential provider API and/or fallback to the clear text value stored in `ssl-server.xml`. Zookeeper-based federation state store and failover controller use Configuration.getPassword to get the Zookeeper authentication info, with fallback provided to clear text auth info.|TODO|
|YARN |WebAppUtils uptakes the use of the credential provider API through the new method on Configuration called getPassword. This provides an alternative to storing the passwords in clear text within the ssl-server.xml file while maintaining backward compatibility. Zookeeper based resource manager state store uses Configuration.getPassword to get the Zookeeper authentication info, with fallback provided to clear text auth info.|TODO|
|KMS |Uses HttpServer2.loadSSLConfiguration that leverages Configuration.getPassword to read SSL related credentials. They may be resolved through Credential Provider and/or from the clear text in the config when allowed.|[KMS](../../hadoop-kms/index.html)|
|HttpFS |Uses HttpServer2.loadSSLConfiguration that leverages Configuration.getPassword to read SSL related credentials. They may be resolved through Credential Provider and/or from the clear text in the config when allowed.|[HttpFS Server Setup](../../hadoop-hdfs-httpfs/ServerSetup.html)|
-|AWS <br/> S3/S3A |Uses Configuration.getPassword to get the S3 credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[AWS S3/S3A Usage](../../hadoop-aws/tools/hadoop-aws/index.html)|
-|Azure <br/> WASB |Uses Configuration.getPassword to get the WASB credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[Azure WASB Usage](../../hadoop-azure/index.html)|
-|Azure <br/> ADLS |Uses Configuration.getPassword to get the ADLS credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[Azure ADLS Usage](../../hadoop-azure-datalake/index.html)|
+|AWS <br/> S3A |Uses `Configuration.getPassword` to get the S3 credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[AWS S3/S3A Usage](../../hadoop-aws/tools/hadoop-aws/index.html)|
+|Azure <br/> WASB |Uses `Configuration.getPassword` to get the WASB credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[Azure WASB Usage](../../hadoop-azure/index.html)|
+|Azure <br/> ADLS |Uses `Configuration.getPassword` to get the ADLS credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[Azure ADLS Usage](../../hadoop-azure-datalake/index.html)|
|Apache <br/> Accumulo|The trace.password property is used by the Tracer to authenticate with Accumulo and persist the traces in the trace table. The credential provider API is used to acquire the trace.password from a provider or from configuration for backward compatibility.|TODO|
|Apache <br/> Slider |A capability has been added to Slider to prompt the user for needed passwords and store them using CredentialProvider so they can be retrieved by an app later.|TODO|
|Apache <br/> Hive |Protection of the metastore password, SSL related passwords and JDO string password has been added through the use of the Credential Provider API|TODO|
@@ -114,13 +116,13 @@ In summary, first, provision the credentials into a provider then configure the
### Credential Management
-#### The hadoop credential Command
+#### The `hadoop credential` Command
Usage: `hadoop credential <subcommand> [options]`
See the command options detail in the [Commands Manual](CommandsManual.html#credential)
-Utilizing the credential command will often be for provisioning a password or secret to a particular credential store provider. In order to explicitly indicate which provider store to use the `-provider` option should be used.
+The credential command can be for provisioning a password or secret to a particular credential store provider. In order to explicitly indicate which provider store to use the `-provider` option should be used.
Example: `hadoop credential create ssl.server.keystore.password -provider jceks://file/tmp/test.jceks`
@@ -128,13 +130,51 @@ In order to indicate a particular provider type and location, the user must prov
#### Provider Types
-1. The `UserProvider`, which is representd by the provider URI `user:///`, is used to retrieve credentials from a user's Credentials file. This file is used to store various tokens, secrets and passwords that are needed by executing jobs and applications.
-2. The `JavaKeyStoreProvider`, which is represented by the provider URI `jceks://file|hdfs/path-to-keystore`, is used to retrieve credentials from a Java keystore. The underlying use of the Hadoop filesystem abstraction allows credentials to be stored on the local filesystem or within HDFS.
+1. The `UserProvider`, which is represented by the provider URI `user:///`, is used to retrieve credentials from a user's Credentials file. This file is used to store various tokens, secrets and passwords that are needed by executing jobs and applications.
+2. The `JavaKeyStoreProvider`, which is represented by the provider URI `jceks://SCHEME/path-to-keystore`, is used to retrieve credentials from a Java keystore file in a filesystem `<SCHEME>`
+ The underlying use of the Hadoop filesystem API allows credentials to be stored on the local filesystem or within cluster stores.
3. The `LocalJavaKeyStoreProvider`, which is represented by the provider URI `localjceks://file/path-to-keystore`, is used to access credentials from a Java keystore that is must be stored on the local filesystem. This is needed for credentials that would result in a recursive dependency on accessing HDFS. Anytime that your credential is required to gain access to HDFS we can't depend on getting a credential out of HDFS to do so.
+When credentials are stored in a filesystem, the following rules apply:
+
+* Credentials stored in local `localjceks://` files are loaded in the process reading in the configuration.
+ For use in a YARN application, this means that they must be visible across the entire cluster, in the local filesystems of the hosts.
+
+* Credentials stored with the `jceks://` provider can be stored in the cluster filesystem,
+and so visible across the cluster —but not in the filesystem which requires the specific
+credentials for their access.
+
+To wrap filesystem URIs with a `jceks` URI follow these steps:
+
+1. Take a filesystem URI such as `hdfs://namenode:9001/users/alice/secrets.jceks`
+1. Place `jceks://` in front of the URL: `jceks://hdfs://namenode:9001/users/alice/secrets.jceks`
+1. Replace the second `://` string with an `@` symbol: `jceks://hdfs@namenode:9001/users/alice/secrets.jceks`
+
+*Examples*
+
+For a local filesystem, a path such as `file:///tmp/secrets.jceks` would become: `jceks://file/tmp/secrets.jceks`
+
+| Path URI | jceks URI |
+|-----------|-----------|
+| `hdfs://namenode.example.org:9001/user/alice/secret.jceks` | `jceks://hdfs@namenode.example.org:9001/user/alice/secret.jceks` |
+| `file:///tmp/secrets.jceks` | `jceks://file/tmp/secret.jceks` |
+| `s3a://container1/secrets/secret.jceks` | `jceks://s3a@container1/secrets/secret.jceks` |
+| `wasb://account@container/secret.jceks` | `jceks://wasb@account@container/secret.jceks` |
+| `abfs://account@container/secret.jceks` | `jceks://abfs@account@container/secret.jceks` |
+| `https://user:pass@service/secret.jceks?token=aia` | `jceks://https@user:pass@service/secret.jceks?token=aia` |
+
+
+Note that to avoid infinite recursion, filesystems such as `abfs`, `s3a`, `adls`
+and `wasb` explicitly exclude keystores stored on paths in their own filesystem
+schemes, even if they are stored in a container which uses a different set of
+credentials from those being looked up.
+
+As an example, you cannot use credentials stored in `s3a://shared/secrets/secret.jceks`
+to read the credentials for the container `s3a://private/` .
+
#### Keystore Passwords
-Keystores in Java are generally protected by passwords. The primary method of protection of the keystore-based credential providers are OS level file permissions and any other policy based access protection that may exist for the target filesystem. While the password is not a primary source of protection, it is very important to understand the mechanics required and options available for managing these passwords. It is also very important to understand all the parties that will need access to the password used to protect the keystores in order to consume them at runtime.
+Keystores in Java are generally protected by passwords. The primary method of protection of the keystore-based credential providers are OS-level file permissions and any other policy based access protection that may exist for the target filesystem. While the password is not a primary source of protection, it is very important to understand the mechanics required and options available for managing these passwords. It is also very important to understand all the parties that will need access to the password used to protect the keystores in order to consume them at runtime.
##### Options
| Option | Description | Notes |
@@ -149,14 +189,34 @@ Extremely important to consider that *all* of the runtime consumers of the crede
|Keystore Password| Description|Sync Required|Clear Text|File Permissions|
|:---- |:---- |:---|:---|:---|
|Default Password|Hardcoded password is the default. Essentially, when using the default password for all keystore-based credential stores, we are leveraging the file permissions to protect the credential store and the keystore password is just a formality of persisting the keystore.|No|Yes|No (documented)|
-|Environment Variable|`HADOOP_CREDSTORE_PASSWORD` Environment variable must be set to the custom password for all keystores that may be configured in the provider path of any process that needs to access credentials from a keystore-based credential provider. There is only one env variable for the entire path of comma separated providers. It is difficult to know the passwords required for each keystore and it is suggested that the same be used for all keystore-based credential providers to avoid this issue. Setting the environment variable will likely require it to be set from a script or some other clear text storage mechanism. Environment variables for running processes are available from various unix commands.|Yes|Yes|No|
+|Environment Variable|The `HADOOP_CREDSTORE_PASSWORD` environment variable must be set to the custom password for all keystores that may be configured in the provider path of any process that needs to access credentials from a keystore-based credential provider. There is only one env variable for the entire path of comma-separated providers. It is difficult to know the passwords required for each keystore and it is suggested that the same be used for all keystore-based credential providers to avoid this issue. Setting the environment variable will likely require it to be set from a script or some other clear text storage mechanism. Environment variables for running processes are available from various unix commands.|Yes|Yes|No|
|Password File|`hadoop.security.credstore.java-keystore-provider.password-file` configuration property must be set to the location of the "side file" that contains the custom password for all keystores that may be configured in the provider path. Any process that needs to access credentials from a keystore-based credential provider will need to have this configuration property set to the appropriate file location. There is only one password-file for the entire path of comma separated providers. It is difficult to know the passwords required for each keystore and it is therefore suggested that the same be used for all keystore-based credential providers to avoid this issue. Password-files are additional files that need to be managed, store the password in clear text and need file permissions to be set such that only those that need access to them have it. If file permissions are set inappropriately the password to access the keystores is available in clear text.|Yes|Yes|Yes|
The use of the default password means that no additional communication/synchronization to runtime consumers needs to be done. The default password is known but file permissions are the primary protection of the keystore.
-When file permissions are thwarted, unlike "side files", there are no standard tools that can expose the protected credentials - even with the password known. Keytool requires a password that is six characters or more and doesn't know how to retrieve general secrets from a keystore. It is also limited to PKI keypairs. Editors will not review the secrets stored within the keystore, nor will `cat`, `more` or any other standard tools. This is why the keystore providers are better than "side file" storage of credentials.
+When file permissions are thwarted, unlike "side files", there are no standard tools that can expose the protected credentials - even with the password known. Keytool requires a password that is six characters or more and doesn't know how to retrieve general secrets from a keystore. It is also limited to PKI keypairs. Editors will not reveal the secrets stored within the keystore, nor will `cat`, `more` or any other standard tools. This is why the keystore providers are better than "side file" storage of credentials.
-That said, it is trivial for someone to write code to access the credentials stored within a keystore-based credential provider using the API. Again, when using the default password, the password is merely a formality of persisting the keystore. The *only* protection is file permissions and OS level access policy.
+That said, it is trivial for someone to write code to access the credentials stored within a keystore-based credential provider using the API. Again, when using the default password, the password is merely a formality of persisting the keystore. The *only* protection is file permissions and OS level access policy.
Users may decide to use a password "side file" to store the password for the keystores themselves and this is supported. It is just really important to be aware of the mechanics required for this level of correctness.
+#### Disabling fallback to plain text
+
+The `Credentials.getPassword()` operation falls back to using entries in the configuration XML files if there are no credential providers, or if a key cannot be found.
+
+This action can be disabled by changing the configuration option `hadoop.security.credential.clear-text-fallback` from `true` to `false`:
+
+```xml
+<property>
+ <name>hadoop.security.credential.clear-text-fallback</name>
+ <value>false</value>
+ <description>
+ true or false to indicate whether or not to fall back to storing credential
+ password as clear text. The default value is true. This property only works
+ when the password can't not be found from credential providers.
+ </description>
+</property>
+```
+
+Once set, *all configuration options looked up via the `getPassword()` API must
+be served via a credential provider*.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d98ca9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
index 9c01175..cb6a1fb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
@@ -27,6 +27,7 @@ import org.junit.Test;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.security.NoSuchAlgorithmException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
@@ -146,14 +147,29 @@ public class TestKeyProvider {
@Test
public void testUnnestUri() throws Exception {
- assertEquals(new Path("hdfs://nn.example.com/my/path"),
- ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn.example.com/my/path")));
- assertEquals(new Path("hdfs://nn/my/path?foo=bar&baz=bat#yyy"),
- ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn/my/path?foo=bar&baz=bat#yyy")));
- assertEquals(new Path("inner://hdfs@nn1.example.com/my/path"),
- ProviderUtils.unnestUri(new URI("outer://inner@hdfs@nn1.example.com/my/path")));
- assertEquals(new Path("user:///"),
- ProviderUtils.unnestUri(new URI("outer://user/")));
+ assertUnwraps("hdfs://nn.example.com/my/path",
+ "myscheme://hdfs@nn.example.com/my/path");
+ assertUnwraps("hdfs://nn/my/path?foo=bar&baz=bat#yyy",
+ "myscheme://hdfs@nn/my/path?foo=bar&baz=bat#yyy");
+ assertUnwraps("inner://hdfs@nn1.example.com/my/path",
+ "outer://inner@hdfs@nn1.example.com/my/path");
+ assertUnwraps("user:///", "outer://user/");
+ assertUnwraps("wasb://account@container/secret.jceks",
+ "jceks://wasb@account@container/secret.jceks");
+ assertUnwraps("abfs://account@container/secret.jceks",
+ "jceks://abfs@account@container/secret.jceks");
+ assertUnwraps("s3a://container/secret.jceks",
+ "jceks://s3a@container/secret.jceks");
+ assertUnwraps("file:///tmp/secret.jceks",
+ "jceks://file/tmp/secret.jceks");
+ assertUnwraps("https://user:pass@service/secret.jceks?token=aia",
+ "jceks://https@user:pass@service/secret.jceks?token=aia");
+ }
+
+ protected void assertUnwraps(final String unwrapped, final String outer)
+ throws URISyntaxException {
+ assertEquals(new Path(unwrapped),
+ ProviderUtils.unnestUri(new URI(outer)));
}
private static class MyKeyProvider extends KeyProvider {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDDS-712. Use x-amz-storage-class
to specify replication type and replication factor. Contributed by Bharat
Viswanadham.
Posted by ae...@apache.org.
HDDS-712. Use x-amz-storage-class to specify replication type and replication factor. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ecac351a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ecac351a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ecac351a
Branch: refs/heads/HDDS-4
Commit: ecac351aac1702194c56743ced5a66242643f28c
Parents: 9c438ab
Author: Márton Elek <el...@apache.org>
Authored: Wed Oct 31 11:08:43 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Wed Oct 31 13:28:59 2018 +0100
----------------------------------------------------------------------
.../dist/src/main/smoketest/s3/awss3.robot | 4 +-
.../dist/src/main/smoketest/s3/objectcopy.robot | 18 ++---
.../src/main/smoketest/s3/objectdelete.robot | 6 +-
.../main/smoketest/s3/objectmultidelete.robot | 6 +-
.../src/main/smoketest/s3/objectputget.robot | 2 +-
.../ozone/s3/endpoint/ObjectEndpoint.java | 68 ++++++++++++++-----
.../apache/hadoop/ozone/s3/util/S3Consts.java | 19 ++++++
.../hadoop/ozone/s3/util/S3StorageType.java | 55 ++++++++++++++++
.../hadoop/ozone/s3/util/package-info.java | 22 +++++++
.../hadoop/ozone/s3/endpoint/TestPutObject.java | 69 +++++++++++---------
10 files changed, 205 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot b/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot
index 79db688..c1ec9f0 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/awss3.robot
@@ -29,9 +29,9 @@ ${BUCKET} generated
File upload and directory list
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}
+ ${result} = Execute AWSS3Cli cp --storage-class REDUCED_REDUNDANCY /tmp/testfile s3://${BUCKET}
Should contain ${result} upload
- ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/dir1/dir2/file
+ ${result} = Execute AWSS3Cli cp --storage-class REDUCED_REDUNDANCY /tmp/testfile s3://${BUCKET}/dir1/dir2/file
Should contain ${result} upload
${result} = Execute AWSS3Cli ls s3://${BUCKET}
Should contain ${result} testfile
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
index 2daa861..e702d9b 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
@@ -39,28 +39,28 @@ Create Dest Bucket
Copy Object Happy Scenario
Run Keyword if '${DESTBUCKET}' == 'generated1' Create Dest Bucket
Execute date > /tmp/copyfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key copyobject/f1 --body /tmp/copyfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key copyobject/f1 --body /tmp/copyfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix copyobject/
Should contain ${result} f1
- ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
+ ${result} = Execute AWSS3ApiCli copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/
Should contain ${result} f1
#copying again will not throw error
- ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
+ ${result} = Execute AWSS3ApiCli copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/
Should contain ${result} f1
Copy Object Where Bucket is not available
- ${result} = Execute AWSS3APICli and checkrc copy-object --bucket dfdfdfdfdfnonexistent --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object --storage-class REDUCED_REDUNDANCY --bucket dfdfdfdfdfnonexistent --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 255
Should contain ${result} NoSuchBucket
- ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source dfdfdfdfdfnonexistent/copyobject/f1 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source dfdfdfdfdfnonexistent/copyobject/f1 255
Should contain ${result} NoSuchBucket
-Copy Object Where both source and dest are same
- ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${DESTBUCKET}/copyobject/f1 255
- Should contain ${result} InvalidRequest
+Copy Object Where both source and dest are same with change to storageclass
+ ${result} = Execute AWSS3APICli copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${DESTBUCKET}/copyobject/f1
+ Should contain ${result} ETag
Copy Object Where Key not available
- ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/nonnonexistentkey 255
+ ${result} = Execute AWSS3APICli and checkrc copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/nonnonexistentkey 255
Should contain ${result} NoSuchKey
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
index 9e57d50..44e1a22 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot
@@ -28,7 +28,7 @@ ${BUCKET} generated
*** Test Cases ***
Delete file with s3api
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key deletetestapi/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key deletetestapi/f1 --body /tmp/testfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix deletetestapi/
Should contain ${result} f1
${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapi/f1
@@ -46,7 +46,7 @@ Delete file with s3api, file doesn't exist
Delete dir with s3api
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/deletetestapidir/f1
+ ${result} = Execute AWSS3Cli cp --storage-class REDUCED_REDUNDANCY /tmp/testfile s3://${BUCKET}/deletetestapidir/f1
${result} = Execute AWSS3Cli ls s3://${BUCKET}/deletetestapidir/
Should contain ${result} f1
${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapidir/
@@ -57,7 +57,7 @@ Delete dir with s3api
Delete file with s3api, file doesn't exist, prefix of a real file
Execute date > /tmp/testfile
- ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/deletetestapiprefix/filefile
+ ${result} = Execute AWSS3Cli cp --storage-class REDUCED_REDUNDANCY /tmp/testfile s3://${BUCKET}/deletetestapiprefix/filefile
${result} = Execute AWSS3Cli ls s3://${BUCKET}/deletetestapiprefix/
Should contain ${result} filefile
${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapiprefix/file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
index 83c967b..e8b7bd3 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
@@ -29,9 +29,9 @@ ${BUCKET} generated
Delete file with multi delete
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f1 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f2 --body /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f3 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key multidelete/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key multidelete/f2 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key multidelete/f3 --body /tmp/testfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix multidelete/
Should contain ${result} multidelete/f1
Should contain ${result} multidelete/f2
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
index 858e472..f650de8 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -30,7 +30,7 @@ ${BUCKET} generated
Put object to s3
Execute date > /tmp/testfile
- ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key putobject/f1 --body /tmp/testfile
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key putobject/f1 --body /tmp/testfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix putobject/
Should contain ${result} f1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 9782d75..7000a3a 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -18,14 +18,12 @@
package org.apache.hadoop.ozone.s3.endpoint;
import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.HEAD;
import javax.ws.rs.HeaderParam;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
@@ -53,11 +51,16 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.ozone.s3.util.S3StorageType;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.util.Time;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+
/**
* Key level rest endpoints.
*/
@@ -91,22 +94,37 @@ public class ObjectEndpoint extends EndpointBase {
public Response put(
@PathParam("bucket") String bucketName,
@PathParam("path") String keyPath,
- @DefaultValue("STAND_ALONE") @QueryParam("replicationType")
- ReplicationType replicationType,
- @DefaultValue("ONE") @QueryParam("replicationFactor")
- ReplicationFactor replicationFactor,
@HeaderParam("Content-Length") long length,
InputStream body) throws IOException, OS3Exception {
OzoneOutputStream output = null;
try {
- String copyHeader = headers.getHeaderString("x-amz-copy-source");
+ String copyHeader = headers.getHeaderString(COPY_SOURCE_HEADER);
+ String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER);
+
+ ReplicationType replicationType;
+ ReplicationFactor replicationFactor;
+ boolean storageTypeDefault;
+ if (storageType == null) {
+ replicationType = S3StorageType.getDefault().getType();
+ replicationFactor = S3StorageType.getDefault().getFactor();
+ storageTypeDefault = true;
+ } else {
+ try {
+ replicationType = S3StorageType.valueOf(storageType).getType();
+ replicationFactor = S3StorageType.valueOf(storageType).getFactor();
+ } catch (IllegalArgumentException ex) {
+ throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
+ storageType);
+ }
+ storageTypeDefault = false;
+ }
if (copyHeader != null) {
//Copy object, as copy source available.
CopyObjectResponse copyObjectResponse = copyObject(
copyHeader, bucketName, keyPath, replicationType,
- replicationFactor);
+ replicationFactor, storageTypeDefault);
return Response.status(Status.OK).entity(copyObjectResponse).header(
"Connection", "close").build();
}
@@ -254,7 +272,8 @@ public class ObjectEndpoint extends EndpointBase {
String destBucket,
String destkey,
ReplicationType replicationType,
- ReplicationFactor replicationFactor)
+ ReplicationFactor replicationFactor,
+ boolean storageTypeDefault)
throws OS3Exception, IOException {
if (copyHeader.startsWith("/")) {
@@ -276,18 +295,35 @@ public class ObjectEndpoint extends EndpointBase {
boolean closed = false;
try {
// Checking whether we trying to copying to it self.
+
if (sourceBucket.equals(destBucket)) {
if (sourceKey.equals(destkey)) {
- OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
- .INVALID_REQUEST, copyHeader);
- ex.setErrorMessage("This copy request is illegal because it is " +
- "trying to copy an object to it self itself without changing " +
- "the object's metadata, storage class, website redirect " +
- "location or encryption attributes.");
- throw ex;
+ // When copying to same storage type when storage type is provided,
+ // we should not throw exception, as aws cli checks if any of the
+ // options like storage type are provided or not when source and
+ // dest are given same
+ if (storageTypeDefault) {
+ OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
+ .INVALID_REQUEST, copyHeader);
+ ex.setErrorMessage("This copy request is illegal because it is " +
+ "trying to copy an object to it self itself without changing " +
+ "the object's metadata, storage class, website redirect " +
+ "location or encryption attributes.");
+ throw ex;
+ } else {
+ // TODO: Actually here we should change storage type, as ozone
+ // still does not support this just returning dummy response
+ // for now
+ CopyObjectResponse copyObjectResponse = new CopyObjectResponse();
+ copyObjectResponse.setETag(OzoneUtils.getRequestID());
+ copyObjectResponse.setLastModified(Instant.ofEpochMilli(
+ Time.now()));
+ return copyObjectResponse;
+ }
}
}
+
OzoneBucket sourceOzoneBucket = getBucket(sourceBucket);
OzoneBucket destOzoneBucket = getBucket(destBucket);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
new file mode 100644
index 0000000..2e7b965
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -0,0 +1,19 @@
+package org.apache.hadoop.ozone.s3.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Set of constants used for S3 implementation.
+ */
+@InterfaceAudience.Private
+public final class S3Consts {
+
+ //Never Constructed
+ private S3Consts() {
+
+ }
+
+ public static final String COPY_SOURCE_HEADER = "x-amz-copy-source";
+ public static final String STORAGE_CLASS_HEADER = "x-amz-storage-class";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
new file mode 100644
index 0000000..c9d1b50
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.util;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+
+/**
+ * Maps S3 storage class values to Ozone replication values.
+ */
+
+public enum S3StorageType {
+
+ REDUCED_REDUNDANCY(ReplicationType.STAND_ALONE, ReplicationFactor.ONE),
+ STANDARD(ReplicationType.RATIS, ReplicationFactor.THREE);
+
+ private final ReplicationType type;
+ private final ReplicationFactor factor;
+
+ S3StorageType(
+ ReplicationType type,
+ ReplicationFactor factor) {
+ this.type = type;
+ this.factor = factor;
+ }
+
+ public ReplicationFactor getFactor() {
+ return factor;
+ }
+
+ public ReplicationType getType() {
+ return type;
+ }
+
+ public static S3StorageType getDefault() {
+ return STANDARD;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/package-info.java
new file mode 100644
index 0000000..af93f08
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains Ozone S3 Util classes.
+ */
+package org.apache.hadoop.ozone.s3.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecac351a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java
index 4f94e56..eb05861 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java
@@ -26,19 +26,21 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.when;
@@ -80,9 +82,9 @@ public class TestPutObject {
objectEndpoint.setHeaders(headers);
//WHEN
- Response response = objectEndpoint.put(bucketName, keyName,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ Response response = objectEndpoint.put(bucketName, keyName, CONTENT
+ .length(), body);
+
//THEN
String volumeName = clientStub.getObjectStore()
@@ -113,8 +115,6 @@ public class TestPutObject {
//WHEN
Response response = objectEndpoint.put(bucketName, keyName,
- ReplicationType.STAND_ALONE,
- ReplicationFactor.ONE,
chunkedContent.length(),
new ByteArrayInputStream(chunkedContent.getBytes()));
@@ -140,8 +140,7 @@ public class TestPutObject {
keyName = "sourceKey";
Response response = objectEndpoint.put(bucketName, keyName,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ CONTENT.length(), body);
String volumeName = clientStub.getObjectStore().getOzoneVolumeName(
bucketName);
@@ -157,12 +156,10 @@ public class TestPutObject {
// Add copy header, and then call put
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
bucketName + "/" + keyName);
- response = objectEndpoint.put(destBucket, destkey,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ response = objectEndpoint.put(destBucket, destkey, CONTENT.length(), body);
// Check destination key and response
volumeName = clientStub.getObjectStore().getOzoneVolumeName(destBucket);
@@ -176,8 +173,7 @@ public class TestPutObject {
// source and dest same
try {
- objectEndpoint.put(bucketName, keyName, ReplicationType.STAND_ALONE,
- ReplicationFactor.ONE, CONTENT.length(), body);
+ objectEndpoint.put(bucketName, keyName, CONTENT.length(), body);
fail("test copy object failed");
} catch (OS3Exception ex) {
Assert.assertTrue(ex.getErrorMessage().contains("This copy request is " +
@@ -186,10 +182,9 @@ public class TestPutObject {
// source bucket not found
try {
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
nonexist + "/" + keyName);
- response = objectEndpoint.put(destBucket, destkey,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
+ objectEndpoint.put(destBucket, destkey, CONTENT.length(),
body);
fail("test copy object failed");
} catch (OS3Exception ex) {
@@ -198,11 +193,9 @@ public class TestPutObject {
// dest bucket not found
try {
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
bucketName + "/" + keyName);
- response = objectEndpoint.put(nonexist, destkey,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ objectEndpoint.put(nonexist, destkey, CONTENT.length(), body);
fail("test copy object failed");
} catch (OS3Exception ex) {
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
@@ -210,11 +203,9 @@ public class TestPutObject {
//Both source and dest bucket not found
try {
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
nonexist + "/" + keyName);
- response = objectEndpoint.put(nonexist, destkey,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ objectEndpoint.put(nonexist, destkey, CONTENT.length(), body);
fail("test copy object failed");
} catch (OS3Exception ex) {
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
@@ -222,15 +213,33 @@ public class TestPutObject {
// source key not found
try {
- when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
bucketName + "/" + nonexist);
- response = objectEndpoint.put("nonexistent", keyName,
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
- body);
+ objectEndpoint.put("nonexistent", keyName, CONTENT.length(), body);
fail("test copy object failed");
} catch (OS3Exception ex) {
Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
}
}
+
+ @Test
+ public void testInvalidStorageType() throws IOException {
+ HttpHeaders headers = Mockito.mock(HttpHeaders.class);
+ ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes());
+ objectEndpoint.setHeaders(headers);
+ keyName = "sourceKey";
+ when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("random");
+
+ try {
+ Response response = objectEndpoint.put(bucketName, keyName,
+ CONTENT.length(), body);
+ fail("testInvalidStorageType");
+ } catch (OS3Exception ex) {
+ assertEquals(S3ErrorTable.INVALID_ARGUMENT.getErrorMessage(),
+ ex.getErrorMessage());
+ assertEquals("random", ex.getResource());
+ }
+
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-8854. Upgrade jquery
datatable version references to v1.10.19. Contributed by Akhil PB.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js
new file mode 100644
index 0000000..3a79ccc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js
@@ -0,0 +1,184 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*!
+ DataTables 1.10.18
+ ©2008-2018 SpryMedia Ltd - datatables.net/license
+*/
+(function(h){"function"===typeof define&&define.amd?define(["jquery"],function(E){return h(E,window,document)}):"object"===typeof exports?module.exports=function(E,H){E||(E=window);H||(H="undefined"!==typeof window?require("jquery"):require("jquery")(E));return h(H,E,E.document)}:h(jQuery,window,document)})(function(h,E,H,k){function Z(a){var b,c,d={};h.each(a,function(e){if((b=e.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=e.replace(b[0],b[2].toLowerCase()),
+d[c]=e,"o"===b[1]&&Z(a[e])});a._hungarianMap=d}function J(a,b,c){a._hungarianMap||Z(a);var d;h.each(b,function(e){d=a._hungarianMap[e];if(d!==k&&(c||b[d]===k))"o"===d.charAt(0)?(b[d]||(b[d]={}),h.extend(!0,b[d],b[e]),J(a[d],b[d],c)):b[d]=b[e]})}function Ca(a){var b=n.defaults.oLanguage,c=b.sDecimal;c&&Da(c);if(a){var d=a.sZeroRecords;!a.sEmptyTable&&(d&&"No data available in table"===b.sEmptyTable)&&F(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(d&&"Loading..."===b.sLoadingRecords)&&F(a,
+a,"sZeroRecords","sLoadingRecords");a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&c!==a&&Da(a)}}function eb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate");A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");"boolean"===typeof a.sScrollX&&(a.sScrollX=a.sScrollX?"100%":
+"");"boolean"===typeof a.scrollX&&(a.scrollX=a.scrollX?"100%":"");if(a=a.aoSearchCols)for(var b=0,c=a.length;b<c;b++)a[b]&&J(n.models.oSearch,a[b])}function fb(a){A(a,"orderable","bSortable");A(a,"orderData","aDataSort");A(a,"orderSequence","asSorting");A(a,"orderDataType","sortDataType");var b=a.aDataSort;"number"===typeof b&&!h.isArray(b)&&(a.aDataSort=[b])}function gb(a){if(!n.__browser){var b={};n.__browser=b;var c=h("<div/>").css({position:"fixed",top:0,left:-1*h(E).scrollLeft(),height:1,width:1,
+overflow:"hidden"}).append(h("<div/>").css({position:"absolute",top:1,left:1,width:100,overflow:"scroll"}).append(h("<div/>").css({width:"100%",height:10}))).appendTo("body"),d=c.children(),e=d.children();b.barWidth=d[0].offsetWidth-d[0].clientWidth;b.bScrollOversize=100===e[0].offsetWidth&&100!==d[0].clientWidth;b.bScrollbarLeft=1!==Math.round(e.offset().left);b.bBounding=c[0].getBoundingClientRect().width?!0:!1;c.remove()}h.extend(a.oBrowser,n.__browser);a.oScroll.iBarWidth=n.__browser.barWidth}
+function hb(a,b,c,d,e,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;d!==e;)a.hasOwnProperty(d)&&(g=j?b(g,a[d],d,a):a[d],j=!0,d+=f);return g}function Ea(a,b){var c=n.defaults.column,d=a.aoColumns.length,c=h.extend({},n.models.oColumn,c,{nTh:b?b:H.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.mData:d,idx:d});a.aoColumns.push(c);c=a.aoPreSearchCols;c[d]=h.extend({},n.models.oSearch,c[d]);ka(a,d,h(b).data())}function ka(a,b,c){var b=a.aoColumns[b],
+d=a.oClasses,e=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=e.attr("width")||null;var f=(e.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(fb(c),J(n.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&&(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),c.sClass&&e.addClass(c.sClass),h.extend(b,c),F(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),F(b,c,"aDataSort"));var g=b.mData,j=S(g),i=b.mRender?
+S(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};b._bAttrSrc=h.isPlainObject(g)&&(c(g.sort)||c(g.type)||c(g.filter));b._setter=null;b.fnGetData=function(a,b,c){var d=j(a,b,k,c);return i&&b?i(d,b,a,c):d};b.fnSetData=function(a,b,c){return N(g)(a,b,c)};"number"!==typeof g&&(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,e.addClass(d.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=d.sSortableNone,
+b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=d.sSortableAsc,b.sSortingClassJUI=d.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=d.sSortableDesc,b.sSortingClassJUI=d.sSortJUIDescAllowed):(b.sSortingClass=d.sSortable,b.sSortingClassJUI=d.sSortJUI)}function $(a){if(!1!==a.oFeatures.bAutoWidth){var b=a.aoColumns;Fa(a);for(var c=0,d=b.length;c<d;c++)b[c].nTh.style.width=b[c].sWidth}b=a.oScroll;(""!==b.sY||""!==b.sX)&&la(a);r(a,null,"column-sizing",[a])}function aa(a,b){var c=ma(a,"bVisible");return"number"===
+typeof c[b]?c[b]:null}function ba(a,b){var c=ma(a,"bVisible"),c=h.inArray(b,c);return-1!==c?c:null}function V(a){var b=0;h.each(a.aoColumns,function(a,d){d.bVisible&&"none"!==h(d.nTh).css("display")&&b++});return b}function ma(a,b){var c=[];h.map(a.aoColumns,function(a,e){a[b]&&c.push(e)});return c}function Ga(a){var b=a.aoColumns,c=a.aoData,d=n.ext.type.detect,e,f,g,j,i,h,l,q,t;e=0;for(f=b.length;e<f;e++)if(l=b[e],t=[],!l.sType&&l._sManualType)l.sType=l._sManualType;else if(!l.sType){g=0;for(j=d.length;g<
+j;g++){i=0;for(h=c.length;i<h;i++){t[i]===k&&(t[i]=B(a,i,e,"type"));q=d[g](t[i],a);if(!q&&g!==d.length-1)break;if("html"===q)break}if(q){l.sType=q;break}}l.sType||(l.sType="string")}}function ib(a,b,c,d){var e,f,g,j,i,m,l=a.aoColumns;if(b)for(e=b.length-1;0<=e;e--){m=b[e];var q=m.targets!==k?m.targets:m.aTargets;h.isArray(q)||(q=[q]);f=0;for(g=q.length;f<g;f++)if("number"===typeof q[f]&&0<=q[f]){for(;l.length<=q[f];)Ea(a);d(q[f],m)}else if("number"===typeof q[f]&&0>q[f])d(l.length+q[f],m);else if("string"===
+typeof q[f]){j=0;for(i=l.length;j<i;j++)("_all"==q[f]||h(l[j].nTh).hasClass(q[f]))&&d(j,m)}}if(c){e=0;for(a=c.length;e<a;e++)d(e,c[e])}}function O(a,b,c,d){var e=a.aoData.length,f=h.extend(!0,{},n.models.oRow,{src:c?"dom":"data",idx:e});f._aData=b;a.aoData.push(f);for(var g=a.aoColumns,j=0,i=g.length;j<i;j++)g[j].sType=null;a.aiDisplayMaster.push(e);b=a.rowIdFn(b);b!==k&&(a.aIds[b]=f);(c||!a.oFeatures.bDeferRender)&&Ha(a,e,c,d);return e}function na(a,b){var c;b instanceof h||(b=h(b));return b.map(function(b,
+e){c=Ia(a,e);return O(a,c.data,e,c.cells)})}function B(a,b,c,d){var e=a.iDraw,f=a.aoColumns[c],g=a.aoData[b]._aData,j=f.sDefaultContent,i=f.fnGetData(g,d,{settings:a,row:b,col:c});if(i===k)return a.iDrawError!=e&&null===j&&(K(a,0,"Requested unknown parameter "+("function"==typeof f.mData?"{function}":"'"+f.mData+"'")+" for row "+b+", column "+c,4),a.iDrawError=e),j;if((i===g||null===i)&&null!==j&&d!==k)i=j;else if("function"===typeof i)return i.call(g);return null===i&&"display"==d?"":i}function jb(a,
+b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d,{settings:a,row:b,col:c})}function Ja(a){return h.map(a.match(/(\\.|[^\.])+/g)||[""],function(a){return a.replace(/\\\./g,".")})}function S(a){if(h.isPlainObject(a)){var b={};h.each(a,function(a,c){c&&(b[a]=S(c))});return function(a,c,f,g){var j=b[c]||b._;return j!==k?j(a,c,f,g):a}}if(null===a)return function(a){return a};if("function"===typeof a)return function(b,c,f,g){return a(b,c,f,g)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||
+-1!==a.indexOf("("))){var c=function(a,b,f){var g,j;if(""!==f){j=Ja(f);for(var i=0,m=j.length;i<m;i++){f=j[i].match(ca);g=j[i].match(W);if(f){j[i]=j[i].replace(ca,"");""!==j[i]&&(a=a[j[i]]);g=[];j.splice(0,i+1);j=j.join(".");if(h.isArray(a)){i=0;for(m=a.length;i<m;i++)g.push(c(a[i],b,j))}a=f[0].substring(1,f[0].length-1);a=""===a?g:g.join(a);break}else if(g){j[i]=j[i].replace(W,"");a=a[j[i]]();continue}if(null===a||a[j[i]]===k)return k;a=a[j[i]]}}return a};return function(b,e){return c(b,e,a)}}return function(b){return b[a]}}
+function N(a){if(h.isPlainObject(a))return N(a._);if(null===a)return function(){};if("function"===typeof a)return function(b,d,e){a(b,"set",d,e)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var b=function(a,d,e){var e=Ja(e),f;f=e[e.length-1];for(var g,j,i=0,m=e.length-1;i<m;i++){g=e[i].match(ca);j=e[i].match(W);if(g){e[i]=e[i].replace(ca,"");a[e[i]]=[];f=e.slice();f.splice(0,i+1);g=f.join(".");if(h.isArray(d)){j=0;for(m=d.length;j<m;j++)f={},b(f,d[j],g),
+a[e[i]].push(f)}else a[e[i]]=d;return}j&&(e[i]=e[i].replace(W,""),a=a[e[i]](d));if(null===a[e[i]]||a[e[i]]===k)a[e[i]]={};a=a[e[i]]}if(f.match(W))a[f.replace(W,"")](d);else a[f.replace(ca,"")]=d};return function(c,d){return b(c,d,a)}}return function(b,d){b[a]=d}}function Ka(a){return D(a.aoData,"_aData")}function oa(a){a.aoData.length=0;a.aiDisplayMaster.length=0;a.aiDisplay.length=0;a.aIds={}}function pa(a,b,c){for(var d=-1,e=0,f=a.length;e<f;e++)a[e]==b?d=e:a[e]>b&&a[e]--; -1!=d&&c===k&&a.splice(d,
+1)}function da(a,b,c,d){var e=a.aoData[b],f,g=function(c,d){for(;c.childNodes.length;)c.removeChild(c.firstChild);c.innerHTML=B(a,b,d,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===e.src)e._aData=Ia(a,e,d,d===k?k:e._aData).data;else{var j=e.anCells;if(j)if(d!==k)g(j[d],d);else{c=0;for(f=j.length;c<f;c++)g(j[c],c)}}e._aSortData=null;e._aFilterData=null;g=a.aoColumns;if(d!==k)g[d].sType=null;else{c=0;for(f=g.length;c<f;c++)g[c].sType=null;La(a,e)}}function Ia(a,b,c,d){var e=[],f=b.firstChild,g,
+j,i=0,m,l=a.aoColumns,q=a._rowReadObject,d=d!==k?d:q?{}:[],t=function(a,b){if("string"===typeof a){var c=a.indexOf("@");-1!==c&&(c=a.substring(c+1),N(a)(d,b.getAttribute(c)))}},G=function(a){if(c===k||c===i)j=l[i],m=h.trim(a.innerHTML),j&&j._bAttrSrc?(N(j.mData._)(d,m),t(j.mData.sort,a),t(j.mData.type,a),t(j.mData.filter,a)):q?(j._setter||(j._setter=N(j.mData)),j._setter(d,m)):d[i]=m;i++};if(f)for(;f;){g=f.nodeName.toUpperCase();if("TD"==g||"TH"==g)G(f),e.push(f);f=f.nextSibling}else{e=b.anCells;
+f=0;for(g=e.length;f<g;f++)G(e[f])}if(b=b.firstChild?b:b.nTr)(b=b.getAttribute("id"))&&N(a.rowId)(d,b);return{data:d,cells:e}}function Ha(a,b,c,d){var e=a.aoData[b],f=e._aData,g=[],j,i,m,l,q;if(null===e.nTr){j=c||H.createElement("tr");e.nTr=j;e.anCells=g;j._DT_RowIndex=b;La(a,e);l=0;for(q=a.aoColumns.length;l<q;l++){m=a.aoColumns[l];i=c?d[l]:H.createElement(m.sCellType);i._DT_CellIndex={row:b,column:l};g.push(i);if((!c||m.mRender||m.mData!==l)&&(!h.isPlainObject(m.mData)||m.mData._!==l+".display"))i.innerHTML=
+B(a,b,l,"display");m.sClass&&(i.className+=" "+m.sClass);m.bVisible&&!c?j.appendChild(i):!m.bVisible&&c&&i.parentNode.removeChild(i);m.fnCreatedCell&&m.fnCreatedCell.call(a.oInstance,i,B(a,b,l),f,b,l)}r(a,"aoRowCreatedCallback",null,[j,f,b,g])}e.nTr.setAttribute("role","row")}function La(a,b){var c=b.nTr,d=b._aData;if(c){var e=a.rowIdFn(d);e&&(c.id=e);d.DT_RowClass&&(e=d.DT_RowClass.split(" "),b.__rowc=b.__rowc?qa(b.__rowc.concat(e)):e,h(c).removeClass(b.__rowc.join(" ")).addClass(d.DT_RowClass));
+d.DT_RowAttr&&h(c).attr(d.DT_RowAttr);d.DT_RowData&&h(c).data(d.DT_RowData)}}function kb(a){var b,c,d,e,f,g=a.nTHead,j=a.nTFoot,i=0===h("th, td",g).length,m=a.oClasses,l=a.aoColumns;i&&(e=h("<tr/>").appendTo(g));b=0;for(c=l.length;b<c;b++)f=l[b],d=h(f.nTh).addClass(f.sClass),i&&d.appendTo(e),a.oFeatures.bSort&&(d.addClass(f.sSortingClass),!1!==f.bSortable&&(d.attr("tabindex",a.iTabIndex).attr("aria-controls",a.sTableId),Ma(a,f.nTh,b))),f.sTitle!=d[0].innerHTML&&d.html(f.sTitle),Na(a,"header")(a,d,
+f,m);i&&ea(a.aoHeader,g);h(g).find(">tr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(m.sHeaderTH);h(j).find(">tr>th, >tr>td").addClass(m.sFooterTH);if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b<c;b++)f=l[b],f.nTf=a[b].cell,f.sClass&&h(f.nTf).addClass(f.sClass)}}function fa(a,b,c){var d,e,f,g=[],j=[],i=a.aoColumns.length,m;if(b){c===k&&(c=!1);d=0;for(e=b.length;d<e;d++){g[d]=b[d].slice();g[d].nTr=b[d].nTr;for(f=i-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[d].splice(f,1);j.push([])}d=
+0;for(e=g.length;d<e;d++){if(a=g[d].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[d].length;f<b;f++)if(m=i=1,j[d][f]===k){a.appendChild(g[d][f].cell);for(j[d][f]=1;g[d+i]!==k&&g[d][f].cell==g[d+i][f].cell;)j[d+i][f]=1,i++;for(;g[d][f+m]!==k&&g[d][f].cell==g[d][f+m].cell;){for(c=0;c<i;c++)j[d+c][f+m]=1;m++}h(g[d][f].cell).attr("rowspan",i).attr("colspan",m)}}}}function P(a){var b=r(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))C(a,!1);else{var b=[],c=0,d=a.asStripeClasses,e=
+d.length,f=a.oLanguage,g=a.iInitDisplayStart,j="ssp"==y(a),i=a.aiDisplay;a.bDrawing=!0;g!==k&&-1!==g&&(a._iDisplayStart=j?g:g>=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=-1);var g=a._iDisplayStart,m=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&!lb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:m;for(j=j?0:g;j<f;j++){var l=i[j],q=a.aoData[l];null===q.nTr&&Ha(a,l);var t=q.nTr;if(0!==e){var G=d[c%e];q._sRowStripe!=G&&(h(t).removeClass(q._sRowStripe).addClass(G),
+q._sRowStripe=G)}r(a,"aoRowCallback",null,[t,q._aData,c,j,l]);b.push(t);c++}}else c=f.sZeroRecords,1==a.iDraw&&"ajax"==y(a)?c=f.sLoadingRecords:f.sEmptyTable&&0===a.fnRecordsTotal()&&(c=f.sEmptyTable),b[0]=h("<tr/>",{"class":e?d[0]:""}).append(h("<td />",{valign:"top",colSpan:V(a),"class":a.oClasses.sRowEmpty}).html(c))[0];r(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Ka(a),g,m,i]);r(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],Ka(a),g,m,i]);d=h(a.nTBody);d.children().detach();
+d.append(h(b));r(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1}}function T(a,b){var c=a.oFeatures,d=c.bFilter;c.bSort&&mb(a);d?ga(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;P(a);a._drawHold=!1}function nb(a){var b=a.oClasses,c=h(a.nTable),c=h("<div/>").insertBefore(c),d=a.oFeatures,e=h("<div/>",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=e[0];a.nTableReinsertBefore=
+a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,m,l,q,k=0;k<f.length;k++){g=null;j=f[k];if("<"==j){i=h("<div/>")[0];m=f[k+1];if("'"==m||'"'==m){l="";for(q=2;f[k+q]!=m;)l+=f[k+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(m=l.split("."),i.id=m[0].substr(1,m[0].length-1),i.className=m[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;k+=q}e.append(i);e=h(i)}else if(">"==j)e=e.parent();else if("l"==j&&d.bPaginate&&d.bLengthChange)g=ob(a);else if("f"==j&&
+d.bFilter)g=pb(a);else if("r"==j&&d.bProcessing)g=qb(a);else if("t"==j)g=rb(a);else if("i"==j&&d.bInfo)g=sb(a);else if("p"==j&&d.bPaginate)g=tb(a);else if(0!==n.ext.feature.length){i=n.ext.feature;q=0;for(m=i.length;q<m;q++)if(j==i[q].cFeature){g=i[q].fnInit(a);break}}g&&(i=a.aanFeatures,i[j]||(i[j]=[]),i[j].push(g),e.append(g))}c.replaceWith(e);a.nHolding=null}function ea(a,b){var c=h(b).children("tr"),d,e,f,g,j,i,m,l,q,k;a.splice(0,a.length);f=0;for(i=c.length;f<i;f++)a.push([]);f=0;for(i=c.length;f<
+i;f++){d=c[f];for(e=d.firstChild;e;){if("TD"==e.nodeName.toUpperCase()||"TH"==e.nodeName.toUpperCase()){l=1*e.getAttribute("colspan");q=1*e.getAttribute("rowspan");l=!l||0===l||1===l?1:l;q=!q||0===q||1===q?1:q;g=0;for(j=a[f];j[g];)g++;m=g;k=1===l?!0:!1;for(j=0;j<l;j++)for(g=0;g<q;g++)a[f+g][m+j]={cell:e,unique:k},a[f+g].nTr=d}e=e.nextSibling}}}function ra(a,b,c){var d=[];c||(c=a.aoHeader,b&&(c=[],ea(c,b)));for(var b=0,e=c.length;b<e;b++)for(var f=0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!d[f]||
+!a.bSortCellsTop))d[f]=c[b][f].cell;return d}function sa(a,b,c){r(a,"aoServerParams","serverParams",[b]);if(b&&h.isArray(b)){var d={},e=/(.*?)\[\]$/;h.each(b,function(a,b){var c=b.name.match(e);c?(c=c[0],d[c]||(d[c]=[]),d[c].push(b.value)):d[b.name]=b.value});b=d}var f,g=a.ajax,j=a.oInstance,i=function(b){r(a,null,"xhr",[a,b,a.jqXHR]);c(b)};if(h.isPlainObject(g)&&g.data){f=g.data;var m="function"===typeof f?f(b,a):f,b="function"===typeof f&&m?m:h.extend(!0,b,m);delete g.data}m={data:b,success:function(b){var c=
+b.error||b.sError;c&&K(a,0,c);a.json=b;i(b)},dataType:"json",cache:!1,type:a.sServerMethod,error:function(b,c){var d=r(a,null,"xhr",[a,null,a.jqXHR]);-1===h.inArray(!0,d)&&("parsererror"==c?K(a,0,"Invalid JSON response",1):4===b.readyState&&K(a,0,"Ajax error",7));C(a,!1)}};a.oAjaxData=b;r(a,null,"preXhr",[a,b]);a.fnServerData?a.fnServerData.call(j,a.sAjaxSource,h.map(b,function(a,b){return{name:b,value:a}}),i,a):a.sAjaxSource||"string"===typeof g?a.jqXHR=h.ajax(h.extend(m,{url:g||a.sAjaxSource})):
+"function"===typeof g?a.jqXHR=g.call(j,b,i,a):(a.jqXHR=h.ajax(h.extend(m,g)),g.data=f)}function lb(a){return a.bAjaxDataGet?(a.iDraw++,C(a,!0),sa(a,ub(a),function(b){vb(a,b)}),!1):!0}function ub(a){var b=a.aoColumns,c=b.length,d=a.oFeatures,e=a.oPreviousSearch,f=a.aoPreSearchCols,g,j=[],i,m,l,k=X(a);g=a._iDisplayStart;i=!1!==d.bPaginate?a._iDisplayLength:-1;var t=function(a,b){j.push({name:a,value:b})};t("sEcho",a.iDraw);t("iColumns",c);t("sColumns",D(b,"sName").join(","));t("iDisplayStart",g);t("iDisplayLength",
+i);var G={draw:a.iDraw,columns:[],order:[],start:g,length:i,search:{value:e.sSearch,regex:e.bRegex}};for(g=0;g<c;g++)m=b[g],l=f[g],i="function"==typeof m.mData?"function":m.mData,G.columns.push({data:i,name:m.sName,searchable:m.bSearchable,orderable:m.bSortable,search:{value:l.sSearch,regex:l.bRegex}}),t("mDataProp_"+g,i),d.bFilter&&(t("sSearch_"+g,l.sSearch),t("bRegex_"+g,l.bRegex),t("bSearchable_"+g,m.bSearchable)),d.bSort&&t("bSortable_"+g,m.bSortable);d.bFilter&&(t("sSearch",e.sSearch),t("bRegex",
+e.bRegex));d.bSort&&(h.each(k,function(a,b){G.order.push({column:b.col,dir:b.dir});t("iSortCol_"+a,b.col);t("sSortDir_"+a,b.dir)}),t("iSortingCols",k.length));b=n.ext.legacy.ajax;return null===b?a.sAjaxSource?j:G:b?j:G}function vb(a,b){var c=ta(a,b),d=b.sEcho!==k?b.sEcho:b.draw,e=b.iTotalRecords!==k?b.iTotalRecords:b.recordsTotal,f=b.iTotalDisplayRecords!==k?b.iTotalDisplayRecords:b.recordsFiltered;if(d){if(1*d<a.iDraw)return;a.iDraw=1*d}oa(a);a._iRecordsTotal=parseInt(e,10);a._iRecordsDisplay=parseInt(f,
+10);d=0;for(e=c.length;d<e;d++)O(a,c[d]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;P(a);a._bInitComplete||ua(a,b);a.bAjaxDataGet=!0;C(a,!1)}function ta(a,b){var c=h.isPlainObject(a.ajax)&&a.ajax.dataSrc!==k?a.ajax.dataSrc:a.sAjaxDataProp;return"data"===c?b.aaData||b[c]:""!==c?S(c)(b):b}function pb(a){var b=a.oClasses,c=a.sTableId,d=a.oLanguage,e=a.oPreviousSearch,f=a.aanFeatures,g='<input type="search" class="'+b.sFilterInput+'"/>',j=d.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_",
+g):j+g,b=h("<div/>",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("<label/>").append(j)),f=function(){var b=!this.value?"":this.value;b!=e.sSearch&&(ga(a,{sSearch:b,bRegex:e.bRegex,bSmart:e.bSmart,bCaseInsensitive:e.bCaseInsensitive}),a._iDisplayStart=0,P(a))},g=null!==a.searchDelay?a.searchDelay:"ssp"===y(a)?400:0,i=h("input",b).val(e.sSearch).attr("placeholder",d.sSearchPlaceholder).on("keyup.DT search.DT input.DT paste.DT cut.DT",g?Oa(f,g):f).on("keypress.DT",function(a){if(13==a.keyCode)return!1}).attr("aria-controls",
+c);h(a.nTable).on("search.dt.DT",function(b,c){if(a===c)try{i[0]!==H.activeElement&&i.val(e.sSearch)}catch(d){}});return b[0]}function ga(a,b,c){var d=a.oPreviousSearch,e=a.aoPreSearchCols,f=function(a){d.sSearch=a.sSearch;d.bRegex=a.bRegex;d.bSmart=a.bSmart;d.bCaseInsensitive=a.bCaseInsensitive};Ga(a);if("ssp"!=y(a)){wb(a,b.sSearch,c,b.bEscapeRegex!==k?!b.bEscapeRegex:b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<e.length;b++)xb(a,e[b].sSearch,b,e[b].bEscapeRegex!==k?!e[b].bEscapeRegex:e[b].bRegex,
+e[b].bSmart,e[b].bCaseInsensitive);yb(a)}else f(b);a.bFiltered=!0;r(a,null,"search",[a])}function yb(a){for(var b=n.ext.search,c=a.aiDisplay,d,e,f=0,g=b.length;f<g;f++){for(var j=[],i=0,m=c.length;i<m;i++)e=c[i],d=a.aoData[e],b[f](a,d._aFilterData,e,d._aData,i)&&j.push(e);c.length=0;h.merge(c,j)}}function xb(a,b,c,d,e,f){if(""!==b){for(var g=[],j=a.aiDisplay,d=Pa(b,d,e,f),e=0;e<j.length;e++)b=a.aoData[j[e]]._aFilterData[c],d.test(b)&&g.push(j[e]);a.aiDisplay=g}}function wb(a,b,c,d,e,f){var d=Pa(b,
+d,e,f),f=a.oPreviousSearch.sSearch,g=a.aiDisplayMaster,j,e=[];0!==n.ext.search.length&&(c=!0);j=zb(a);if(0>=b.length)a.aiDisplay=g.slice();else{if(j||c||f.length>b.length||0!==b.indexOf(f)||a.bSorted)a.aiDisplay=g.slice();b=a.aiDisplay;for(c=0;c<b.length;c++)d.test(a.aoData[b[c]]._sFilterRow)&&e.push(b[c]);a.aiDisplay=e}}function Pa(a,b,c,d){a=b?a:Qa(a);c&&(a="^(?=.*?"+h.map(a.match(/"[^"]+"|[^ ]+/g)||[""],function(a){if('"'===a.charAt(0))var b=a.match(/^"(.*)"$/),a=b?b[1]:a;return a.replace('"',
+"")}).join(")(?=.*?")+").*$");return RegExp(a,d?"i":"")}function zb(a){var b=a.aoColumns,c,d,e,f,g,j,i,h,l=n.ext.type.search;c=!1;d=0;for(f=a.aoData.length;d<f;d++)if(h=a.aoData[d],!h._aFilterData){j=[];e=0;for(g=b.length;e<g;e++)c=b[e],c.bSearchable?(i=B(a,d,e,"filter"),l[c.sType]&&(i=l[c.sType](i)),null===i&&(i=""),"string"!==typeof i&&i.toString&&(i=i.toString())):i="",i.indexOf&&-1!==i.indexOf("&")&&(va.innerHTML=i,i=Wb?va.textContent:va.innerText),i.replace&&(i=i.replace(/[\r\n]/g,"")),j.push(i);
+h._aFilterData=j;h._sFilterRow=j.join(" ");c=!0}return c}function Ab(a){return{search:a.sSearch,smart:a.bSmart,regex:a.bRegex,caseInsensitive:a.bCaseInsensitive}}function Bb(a){return{sSearch:a.search,bSmart:a.smart,bRegex:a.regex,bCaseInsensitive:a.caseInsensitive}}function sb(a){var b=a.sTableId,c=a.aanFeatures.i,d=h("<div/>",{"class":a.oClasses.sInfo,id:!c?b+"_info":null});c||(a.aoDrawCallback.push({fn:Cb,sName:"information"}),d.attr("role","status").attr("aria-live","polite"),h(a.nTable).attr("aria-describedby",
+b+"_info"));return d[0]}function Cb(a){var b=a.aanFeatures.i;if(0!==b.length){var c=a.oLanguage,d=a._iDisplayStart+1,e=a.fnDisplayEnd(),f=a.fnRecordsTotal(),g=a.fnRecordsDisplay(),j=g?c.sInfo:c.sInfoEmpty;g!==f&&(j+=" "+c.sInfoFiltered);j+=c.sInfoPostFix;j=Db(a,j);c=c.fnInfoCallback;null!==c&&(j=c.call(a.oInstance,a,d,e,f,g,j));h(b).html(j)}}function Db(a,b){var c=a.fnFormatNumber,d=a._iDisplayStart+1,e=a._iDisplayLength,f=a.fnRecordsDisplay(),g=-1===e;return b.replace(/_START_/g,c.call(a,d)).replace(/_END_/g,
+c.call(a,a.fnDisplayEnd())).replace(/_MAX_/g,c.call(a,a.fnRecordsTotal())).replace(/_TOTAL_/g,c.call(a,f)).replace(/_PAGE_/g,c.call(a,g?1:Math.ceil(d/e))).replace(/_PAGES_/g,c.call(a,g?1:Math.ceil(f/e)))}function ha(a){var b,c,d=a.iInitDisplayStart,e=a.aoColumns,f;c=a.oFeatures;var g=a.bDeferLoading;if(a.bInitialised){nb(a);kb(a);fa(a,a.aoHeader);fa(a,a.aoFooter);C(a,!0);c.bAutoWidth&&Fa(a);b=0;for(c=e.length;b<c;b++)f=e[b],f.sWidth&&(f.nTh.style.width=v(f.sWidth));r(a,null,"preInit",[a]);T(a);e=
+y(a);if("ssp"!=e||g)"ajax"==e?sa(a,[],function(c){var f=ta(a,c);for(b=0;b<f.length;b++)O(a,f[b]);a.iInitDisplayStart=d;T(a);C(a,!1);ua(a,c)},a):(C(a,!1),ua(a))}else setTimeout(function(){ha(a)},200)}function ua(a,b){a._bInitComplete=!0;(b||a.oInit.aaData)&&$(a);r(a,null,"plugin-init",[a,b]);r(a,"aoInitComplete","init",[a,b])}function Ra(a,b){var c=parseInt(b,10);a._iDisplayLength=c;Sa(a);r(a,null,"length",[a,c])}function ob(a){for(var b=a.oClasses,c=a.sTableId,d=a.aLengthMenu,e=h.isArray(d[0]),f=
+e?d[0]:d,d=e?d[1]:d,e=h("<select/>",{name:c+"_length","aria-controls":c,"class":b.sLengthSelect}),g=0,j=f.length;g<j;g++)e[0][g]=new Option("number"===typeof d[g]?a.fnFormatNumber(d[g]):d[g],f[g]);var i=h("<div><label/></div>").addClass(b.sLength);a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",e[0].outerHTML));h("select",i).val(a._iDisplayLength).on("change.DT",function(){Ra(a,h(this).val());P(a)});h(a.nTable).on("length.dt.DT",function(b,c,d){a===
+c&&h("select",i).val(d)});return i[0]}function tb(a){var b=a.sPaginationType,c=n.ext.pager[b],d="function"===typeof c,e=function(a){P(a)},b=h("<div/>").addClass(a.oClasses.sPaging+b)[0],f=a.aanFeatures;d||c.fnInit(a,b,e);f.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(a){if(d){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),k,l=0;for(k=f.p.length;l<k;l++)Na(a,"pageButton")(a,f.p[l],l,h,b,i)}else c.fnUpdate(a,
+e)},sName:"pagination"}));return b}function Ta(a,b,c){var d=a._iDisplayStart,e=a._iDisplayLength,f=a.fnRecordsDisplay();0===f||-1===e?d=0:"number"===typeof b?(d=b*e,d>f&&(d=0)):"first"==b?d=0:"previous"==b?(d=0<=e?d-e:0,0>d&&(d=0)):"next"==b?d+e<f&&(d+=e):"last"==b?d=Math.floor((f-1)/e)*e:K(a,0,"Unknown paging action: "+b,5);b=a._iDisplayStart!==d;a._iDisplayStart=d;b&&(r(a,null,"page",[a]),c&&P(a));return b}function qb(a){return h("<div/>",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]}
+function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");r(a,null,"processing",[a,b])}function rb(a){var b=h(a.nTable);b.attr("role","grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var d=c.sX,e=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),m=h(b[0].cloneNode(!1)),l=b.children("tfoot");l.length||(l=null);i=h("<div/>",{"class":f.sScrollWrapper}).append(h("<div/>",{"class":f.sScrollHead}).css({overflow:"hidden",
+position:"relative",border:0,width:d?!d?null:v(d):"100%"}).append(h("<div/>",{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("<div/>",{"class":f.sScrollBody}).css({position:"relative",overflow:"auto",width:!d?null:v(d)}).append(b));l&&i.append(h("<div/>",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:d?!d?null:v(d):"100%"}).append(h("<div/>",
+{"class":f.sScrollFootInner}).append(m.removeAttr("id").css("margin-left",0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=i.children(),k=b[0],f=b[1],t=l?b[2]:null;if(d)h(f).on("scroll.DT",function(){var a=this.scrollLeft;k.scrollLeft=a;l&&(t.scrollLeft=a)});h(f).css(e&&c.bCollapse?"max-height":"height",e);a.nScrollHead=k;a.nScrollBody=f;a.nScrollFoot=t;a.aoDrawCallback.push({fn:la,sName:"scrolling"});return i[0]}function la(a){var b=a.oScroll,c=b.sX,d=b.sXInner,e=b.sY,b=b.iBarWidth,
+f=h(a.nScrollHead),g=f[0].style,j=f.children("div"),i=j[0].style,m=j.children("table"),j=a.nScrollBody,l=h(j),q=j.style,t=h(a.nScrollFoot).children("div"),n=t.children("table"),o=h(a.nTHead),p=h(a.nTable),s=p[0],r=s.style,u=a.nTFoot?h(a.nTFoot):null,x=a.oBrowser,U=x.bScrollOversize,Xb=D(a.aoColumns,"nTh"),Q,L,R,w,Ua=[],y=[],z=[],A=[],B,C=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};L=j.scrollHeight>j.clientHeight;if(a.scrollBarVis!==
+L&&a.scrollBarVis!==k)a.scrollBarVis=L,$(a);else{a.scrollBarVis=L;p.children("thead, tfoot").remove();u&&(R=u.clone().prependTo(p),Q=u.find("tr"),R=R.find("tr"));w=o.clone().prependTo(p);o=o.find("tr");L=w.find("tr");w.find("th, td").removeAttr("tabindex");c||(q.width="100%",f[0].style.width="100%");h.each(ra(a,w),function(b,c){B=aa(a,b);c.style.width=a.aoColumns[B].sWidth});u&&I(function(a){a.style.width=""},R);f=p.outerWidth();if(""===c){r.width="100%";if(U&&(p.find("tbody").height()>j.offsetHeight||
+"scroll"==l.css("overflow-y")))r.width=v(p.outerWidth()-b);f=p.outerWidth()}else""!==d&&(r.width=v(d),f=p.outerWidth());I(C,L);I(function(a){z.push(a.innerHTML);Ua.push(v(h(a).css("width")))},L);I(function(a,b){if(h.inArray(a,Xb)!==-1)a.style.width=Ua[b]},o);h(L).height(0);u&&(I(C,R),I(function(a){A.push(a.innerHTML);y.push(v(h(a).css("width")))},R),I(function(a,b){a.style.width=y[b]},Q),h(R).height(0));I(function(a,b){a.innerHTML='<div class="dataTables_sizing">'+z[b]+"</div>";a.childNodes[0].style.height=
+"0";a.childNodes[0].style.overflow="hidden";a.style.width=Ua[b]},L);u&&I(function(a,b){a.innerHTML='<div class="dataTables_sizing">'+A[b]+"</div>";a.childNodes[0].style.height="0";a.childNodes[0].style.overflow="hidden";a.style.width=y[b]},R);if(p.outerWidth()<f){Q=j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")?f+b:f;if(U&&(j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")))r.width=v(Q-b);(""===c||""!==d)&&K(a,1,"Possible column misalignment",6)}else Q="100%";q.width=v(Q);
+g.width=v(Q);u&&(a.nScrollFoot.style.width=v(Q));!e&&U&&(q.height=v(s.offsetHeight+b));c=p.outerWidth();m[0].style.width=v(c);i.width=v(c);d=p.height()>j.clientHeight||"scroll"==l.css("overflow-y");e="padding"+(x.bScrollbarLeft?"Left":"Right");i[e]=d?b+"px":"0px";u&&(n[0].style.width=v(c),t[0].style.width=v(c),t[0].style[e]=d?b+"px":"0px");p.children("colgroup").insertBefore(p.children("thead"));l.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)j.scrollTop=0}}function I(a,b,c){for(var d=0,e=0,
+f=b.length,g,j;e<f;){g=b[e].firstChild;for(j=c?c[e].firstChild:null;g;)1===g.nodeType&&(c?a(g,j,d):a(g,d),d++),g=g.nextSibling,j=c?j.nextSibling:null;e++}}function Fa(a){var b=a.nTable,c=a.aoColumns,d=a.oScroll,e=d.sY,f=d.sX,g=d.sXInner,j=c.length,i=ma(a,"bVisible"),m=h("th",a.nTHead),l=b.getAttribute("width"),k=b.parentNode,t=!1,n,o,p=a.oBrowser,d=p.bScrollOversize;(n=b.style.width)&&-1!==n.indexOf("%")&&(l=n);for(n=0;n<i.length;n++)o=c[i[n]],null!==o.sWidth&&(o.sWidth=Eb(o.sWidthOrig,k),t=!0);if(d||
+!t&&!f&&!e&&j==V(a)&&j==m.length)for(n=0;n<j;n++)i=aa(a,n),null!==i&&(c[i].sWidth=v(m.eq(n).width()));else{j=h(b).clone().css("visibility","hidden").removeAttr("id");j.find("tbody tr").remove();var s=h("<tr/>").appendTo(j.find("tbody"));j.find("thead, tfoot").remove();j.append(h(a.nTHead).clone()).append(h(a.nTFoot).clone());j.find("tfoot th, tfoot td").css("width","");m=ra(a,j.find("thead")[0]);for(n=0;n<i.length;n++)o=c[i[n]],m[n].style.width=null!==o.sWidthOrig&&""!==o.sWidthOrig?v(o.sWidthOrig):
+"",o.sWidthOrig&&f&&h(m[n]).append(h("<div/>").css({width:o.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(a.aoData.length)for(n=0;n<i.length;n++)t=i[n],o=c[t],h(Fb(a,t)).clone(!1).append(o.sContentPadding).appendTo(s);h("[name]",j).removeAttr("name");o=h("<div/>").css(f||e?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(j).appendTo(k);f&&g?j.width(g):f?(j.css("width","auto"),j.removeAttr("width"),j.width()<k.clientWidth&&l&&j.width(k.clientWidth)):e?j.width(k.clientWidth):
+l&&j.width(l);for(n=e=0;n<i.length;n++)k=h(m[n]),g=k.outerWidth()-k.width(),k=p.bBounding?Math.ceil(m[n].getBoundingClientRect().width):k.outerWidth(),e+=k,c[i[n]].sWidth=v(k-g);b.style.width=v(e);o.remove()}l&&(b.style.width=v(l));if((l||f)&&!a._reszEvt)b=function(){h(E).on("resize.DT-"+a.sInstance,Oa(function(){$(a)}))},d?setTimeout(b,1E3):b(),a._reszEvt=!0}function Eb(a,b){if(!a)return 0;var c=h("<div/>").css("width",v(a)).appendTo(b||H.body),d=c[0].offsetWidth;c.remove();return d}function Fb(a,
+b){var c=Gb(a,b);if(0>c)return null;var d=a.aoData[c];return!d.nTr?h("<td/>").html(B(a,c,b,"display"))[0]:d.anCells[b]}function Gb(a,b){for(var c,d=-1,e=-1,f=0,g=a.aoData.length;f<g;f++)c=B(a,f,b,"display")+"",c=c.replace(Yb,""),c=c.replace(/ /g," "),c.length>d&&(d=c.length,e=f);return e}function v(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function X(a){var b,c,d=[],e=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var m=[];f=function(a){a.length&&
+!h.isArray(a[0])?m.push(a):h.merge(m,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;a<m.length;a++){i=m[a][0];f=e[i].aDataSort;b=0;for(c=f.length;b<c;b++)g=f[b],j=e[g].sType||"string",m[a]._idx===k&&(m[a]._idx=h.inArray(m[a][1],e[g].asSorting)),d.push({src:i,col:g,dir:m[a][1],index:m[a]._idx,type:j,formatter:n.ext.type.order[j+"-pre"]})}return d}function mb(a){var b,c,d=[],e=n.ext.type.order,f=a.aoData,g=0,j,i=a.aiDisplayMaster,h;Ga(a);h=X(a);b=0;for(c=h.length;b<
+c;b++)j=h[b],j.formatter&&g++,Hb(a,j.col);if("ssp"!=y(a)&&0!==h.length){b=0;for(c=i.length;b<c;b++)d[i[b]]=b;g===h.length?i.sort(function(a,b){var c,e,g,j,i=h.length,k=f[a]._aSortData,n=f[b]._aSortData;for(g=0;g<i;g++)if(j=h[g],c=k[j.col],e=n[j.col],c=c<e?-1:c>e?1:0,0!==c)return"asc"===j.dir?c:-c;c=d[a];e=d[b];return c<e?-1:c>e?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,n=f[a]._aSortData,o=f[b]._aSortData;for(j=0;j<k;j++)if(i=h[j],c=n[i.col],g=o[i.col],i=e[i.type+"-"+i.dir]||e["string-"+i.dir],
+c=i(c,g),0!==c)return c;c=d[a];g=d[b];return c<g?-1:c>g?1:0})}a.bSorted=!0}function Ib(a){for(var b,c,d=a.aoColumns,e=X(a),a=a.oLanguage.oAria,f=0,g=d.length;f<g;f++){c=d[f];var j=c.asSorting;b=c.sTitle.replace(/<.*?>/g,"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0<e.length&&e[0].col==f?(i.setAttribute("aria-sort","asc"==e[0].dir?"ascending":"descending"),c=j[e[0].index+1]||j[0]):c=j[0],b+="asc"===c?a.sSortAscending:a.sSortDescending);i.setAttribute("aria-label",b)}}function Va(a,
+b,c,d){var e=a.aaSorting,f=a.aoColumns[b].asSorting,g=function(a,b){var c=a._idx;c===k&&(c=h.inArray(a[1],f));return c+1<f.length?c+1:b?null:0};"number"===typeof e[0]&&(e=a.aaSorting=[e]);c&&a.oFeatures.bSortMulti?(c=h.inArray(b,D(e,"0")),-1!==c?(b=g(e[c],!0),null===b&&1===e.length&&(b=0),null===b?e.splice(c,1):(e[c][1]=f[b],e[c]._idx=b)):(e.push([b,f[0],0]),e[e.length-1]._idx=0)):e.length&&e[0][0]==b?(b=g(e[0]),e.length=1,e[0][1]=f[b],e[0]._idx=b):(e.length=0,e.push([b,f[0]]),e[0]._idx=0);T(a);"function"==
+typeof d&&d(a)}function Ma(a,b,c,d){var e=a.aoColumns[c];Wa(b,{},function(b){!1!==e.bSortable&&(a.oFeatures.bProcessing?(C(a,!0),setTimeout(function(){Va(a,c,b.shiftKey,d);"ssp"!==y(a)&&C(a,!1)},0)):Va(a,c,b.shiftKey,d))})}function wa(a){var b=a.aLastSort,c=a.oClasses.sSortColumn,d=X(a),e=a.oFeatures,f,g;if(e.bSort&&e.bSortClasses){e=0;for(f=b.length;e<f;e++)g=b[e].src,h(D(a.aoData,"anCells",g)).removeClass(c+(2>e?e+1:3));e=0;for(f=d.length;e<f;e++)g=d[e].src,h(D(a.aoData,"anCells",g)).addClass(c+
+(2>e?e+1:3))}a.aLastSort=d}function Hb(a,b){var c=a.aoColumns[b],d=n.ext.order[c.sSortDataType],e;d&&(e=d.call(a.oInstance,a,b,ba(a,b)));for(var f,g=n.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j<i;j++)if(c=a.aoData[j],c._aSortData||(c._aSortData=[]),!c._aSortData[b]||d)f=d?e[j]:B(a,j,b,"sort"),c._aSortData[b]=g?g(f):f}function xa(a){if(a.oFeatures.bStateSave&&!a.bDestroying){var b={time:+new Date,start:a._iDisplayStart,length:a._iDisplayLength,order:h.extend(!0,[],a.aaSorting),search:Ab(a.oPreviousSearch),
+columns:h.map(a.aoColumns,function(b,d){return{visible:b.bVisible,search:Ab(a.aoPreSearchCols[d])}})};r(a,"aoStateSaveParams","stateSaveParams",[a,b]);a.oSavedState=b;a.fnStateSaveCallback.call(a.oInstance,a,b)}}function Jb(a,b,c){var d,e,f=a.aoColumns,b=function(b){if(b&&b.time){var g=r(a,"aoStateLoadParams","stateLoadParams",[a,b]);if(-1===h.inArray(!1,g)&&(g=a.iStateDuration,!(0<g&&b.time<+new Date-1E3*g)&&!(b.columns&&f.length!==b.columns.length))){a.oLoadedState=h.extend(!0,{},b);b.start!==k&&
+(a._iDisplayStart=b.start,a.iInitDisplayStart=b.start);b.length!==k&&(a._iDisplayLength=b.length);b.order!==k&&(a.aaSorting=[],h.each(b.order,function(b,c){a.aaSorting.push(c[0]>=f.length?[0,c[1]]:c)}));b.search!==k&&h.extend(a.oPreviousSearch,Bb(b.search));if(b.columns){d=0;for(e=b.columns.length;d<e;d++)g=b.columns[d],g.visible!==k&&(f[d].bVisible=g.visible),g.search!==k&&h.extend(a.aoPreSearchCols[d],Bb(g.search))}r(a,"aoStateLoaded","stateLoaded",[a,b])}}c()};if(a.oFeatures.bStateSave){var g=
+a.fnStateLoadCallback.call(a.oInstance,a,b);g!==k&&b(g)}else c()}function ya(a){var b=n.settings,a=h.inArray(a,D(b,"nTable"));return-1!==a?b[a]:null}function K(a,b,c,d){c="DataTables warning: "+(a?"table id="+a.sTableId+" - ":"")+c;d&&(c+=". For more information about this error, please see http://datatables.net/tn/"+d);if(b)E.console&&console.log&&console.log(c);else if(b=n.ext,b=b.sErrMode||b.errMode,a&&r(a,null,"error",[a,d,c]),"alert"==b)alert(c);else{if("throw"==b)throw Error(c);"function"==
+typeof b&&b(a,d,c)}}function F(a,b,c,d){h.isArray(c)?h.each(c,function(c,d){h.isArray(d)?F(a,b,d[0],d[1]):F(a,b,d)}):(d===k&&(d=c),b[c]!==k&&(a[d]=b[c]))}function Xa(a,b,c){var d,e;for(e in b)b.hasOwnProperty(e)&&(d=b[e],h.isPlainObject(d)?(h.isPlainObject(a[e])||(a[e]={}),h.extend(!0,a[e],d)):a[e]=c&&"data"!==e&&"aaData"!==e&&h.isArray(d)?d.slice():d);return a}function Wa(a,b,c){h(a).on("click.DT",b,function(b){h(a).blur();c(b)}).on("keypress.DT",b,function(a){13===a.which&&(a.preventDefault(),c(a))}).on("selectstart.DT",
+function(){return!1})}function z(a,b,c,d){c&&a[b].push({fn:c,sName:d})}function r(a,b,c,d){var e=[];b&&(e=h.map(a[b].slice().reverse(),function(b){return b.fn.apply(a.oInstance,d)}));null!==c&&(b=h.Event(c+".dt"),h(a.nTable).trigger(b,d),e.push(b.result));return e}function Sa(a){var b=a._iDisplayStart,c=a.fnDisplayEnd(),d=a._iDisplayLength;b>=c&&(b=c-d);b-=b%d;if(-1===d||0>b)b=0;a._iDisplayStart=b}function Na(a,b){var c=a.renderer,d=n.ext.renderer[b];return h.isPlainObject(c)&&c[b]?d[c[b]]||d._:"string"===
+typeof c?d[c]||d._:d._}function y(a){return a.oFeatures.bServerSide?"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function ia(a,b){var c=[],c=Kb.numbers_length,d=Math.floor(c/2);b<=c?c=Y(0,b):a<=d?(c=Y(0,c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-d?c=Y(b-(c-2),b):(c=Y(a-d+2,a+d-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function Da(a){h.each({num:function(b){return za(b,a)},"num-fmt":function(b){return za(b,a,Ya)},"html-num":function(b){return za(b,
+a,Aa)},"html-num-fmt":function(b){return za(b,a,Aa,Ya)}},function(b,c){x.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(x.type.search[b+a]=x.type.search.html)})}function Lb(a){return function(){var b=[ya(this[n.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return n.ext.internal[a].apply(this,b)}}var n=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new s(ya(this[x.iApiIndex])):new s(this)};
+this.fnAddData=function(a,b){var c=this.api(!0),d=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b===k||b)&&c.draw();return d.flatten().toArray()};this.fnAdjustColumnSizing=function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],d=c.oScroll;a===k||a?b.draw(!1):(""!==d.sX||""!==d.sY)&&la(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a,
+b,c){var d=this.api(!0),a=d.rows(a),e=a.settings()[0],h=e.aoData[a[0][0]];a.remove();b&&b.call(this,e,h);(c===k||c)&&d.draw();return h};this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};this.fnFilter=function(a,b,c,d,e,h){e=this.api(!0);null===b||b===k?e.search(a,c,d,h):e.column(b).search(a,c,d,h);e.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var d=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==d||"th"==d?c.cell(a,b).data():
+c.row(a).data()||null}return c.data().toArray()};this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()};this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]};
+this.fnPageChange=function(a,b){var c=this.api(!0).page(a);(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c===k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return ya(this[x.iApiIndex])};this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,d,e){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(e===k||e)&&h.columns.adjust();
+(d===k||d)&&h.draw();return 0};this.fnVersionCheck=x.fnVersionCheck;var b=this,c=a===k,d=this.length;c&&(a={});this.oApi=this.internal=x.internal;for(var e in n.ext.internal)e&&(this[e]=Lb(e));this.each(function(){var e={},g=1<d?Xa(e,a,!0):a,j=0,i,e=this.getAttribute("id"),m=!1,l=n.defaults,q=h(this);if("table"!=this.nodeName.toLowerCase())K(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{eb(l);fb(l.column);J(l,l,!0);J(l.column,l.column,!0);J(l,h.extend(g,q.data()));var t=n.settings,
+j=0;for(i=t.length;j<i;j++){var o=t[j];if(o.nTable==this||o.nTHead&&o.nTHead.parentNode==this||o.nTFoot&&o.nTFoot.parentNode==this){var s=g.bRetrieve!==k?g.bRetrieve:l.bRetrieve;if(c||s)return o.oInstance;if(g.bDestroy!==k?g.bDestroy:l.bDestroy){o.oInstance.fnDestroy();break}else{K(o,0,"Cannot reinitialise DataTable",3);return}}if(o.sTableId==this.id){t.splice(j,1);break}}if(null===e||""===e)this.id=e="DataTables_Table_"+n.ext._unique++;var p=h.extend(!0,{},n.models.oSettings,{sDestroyWidth:q[0].style.width,
+sInstance:e,sTableId:e});p.nTable=this;p.oApi=b.internal;p.oInit=g;t.push(p);p.oInstance=1===b.length?b:q.dataTable();eb(g);Ca(g.oLanguage);g.aLengthMenu&&!g.iDisplayLength&&(g.iDisplayLength=h.isArray(g.aLengthMenu[0])?g.aLengthMenu[0][0]:g.aLengthMenu[0]);g=Xa(h.extend(!0,{},l),g);F(p.oFeatures,g,"bPaginate bLengthChange bFilter bSort bSortMulti bInfo bProcessing bAutoWidth bSortClasses bServerSide bDeferRender".split(" "));F(p,g,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod",
+"aaSorting","aaSortingFixed","aLengthMenu","sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback","renderer","searchDelay","rowId",["iCookieDuration","iStateDuration"],["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"]]);F(p.oScroll,g,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]);F(p.oLanguage,g,"fnInfoCallback");
+z(p,"aoDrawCallback",g.fnDrawCallback,"user");z(p,"aoServerParams",g.fnServerParams,"user");z(p,"aoStateSaveParams",g.fnStateSaveParams,"user");z(p,"aoStateLoadParams",g.fnStateLoadParams,"user");z(p,"aoStateLoaded",g.fnStateLoaded,"user");z(p,"aoRowCallback",g.fnRowCallback,"user");z(p,"aoRowCreatedCallback",g.fnCreatedRow,"user");z(p,"aoHeaderCallback",g.fnHeaderCallback,"user");z(p,"aoFooterCallback",g.fnFooterCallback,"user");z(p,"aoInitComplete",g.fnInitComplete,"user");z(p,"aoPreDrawCallback",
+g.fnPreDrawCallback,"user");p.rowIdFn=S(g.rowId);gb(p);var u=p.oClasses;h.extend(u,n.ext.classes,g.oClasses);q.addClass(u.sTable);p.iInitDisplayStart===k&&(p.iInitDisplayStart=g.iDisplayStart,p._iDisplayStart=g.iDisplayStart);null!==g.iDeferLoading&&(p.bDeferLoading=!0,e=h.isArray(g.iDeferLoading),p._iRecordsDisplay=e?g.iDeferLoading[0]:g.iDeferLoading,p._iRecordsTotal=e?g.iDeferLoading[1]:g.iDeferLoading);var v=p.oLanguage;h.extend(!0,v,g.oLanguage);v.sUrl&&(h.ajax({dataType:"json",url:v.sUrl,success:function(a){Ca(a);
+J(l.oLanguage,a);h.extend(true,v,a);ha(p)},error:function(){ha(p)}}),m=!0);null===g.asStripeClasses&&(p.asStripeClasses=[u.sStripeOdd,u.sStripeEven]);var e=p.asStripeClasses,x=q.children("tbody").find("tr").eq(0);-1!==h.inArray(!0,h.map(e,function(a){return x.hasClass(a)}))&&(h("tbody tr",this).removeClass(e.join(" ")),p.asDestroyStripes=e.slice());e=[];t=this.getElementsByTagName("thead");0!==t.length&&(ea(p.aoHeader,t[0]),e=ra(p));if(null===g.aoColumns){t=[];j=0;for(i=e.length;j<i;j++)t.push(null)}else t=
+g.aoColumns;j=0;for(i=t.length;j<i;j++)Ea(p,e?e[j]:null);ib(p,g.aoColumnDefs,t,function(a,b){ka(p,a,b)});if(x.length){var w=function(a,b){return a.getAttribute("data-"+b)!==null?b:null};h(x[0]).children("th, td").each(function(a,b){var c=p.aoColumns[a];if(c.mData===a){var d=w(b,"sort")||w(b,"order"),e=w(b,"filter")||w(b,"search");if(d!==null||e!==null){c.mData={_:a+".display",sort:d!==null?a+".@data-"+d:k,type:d!==null?a+".@data-"+d:k,filter:e!==null?a+".@data-"+e:k};ka(p,a)}}})}var U=p.oFeatures,
+e=function(){if(g.aaSorting===k){var a=p.aaSorting;j=0;for(i=a.length;j<i;j++)a[j][1]=p.aoColumns[j].asSorting[0]}wa(p);U.bSort&&z(p,"aoDrawCallback",function(){if(p.bSorted){var a=X(p),b={};h.each(a,function(a,c){b[c.src]=c.dir});r(p,null,"order",[p,a,b]);Ib(p)}});z(p,"aoDrawCallback",function(){(p.bSorted||y(p)==="ssp"||U.bDeferRender)&&wa(p)},"sc");var a=q.children("caption").each(function(){this._captionSide=h(this).css("caption-side")}),b=q.children("thead");b.length===0&&(b=h("<thead/>").appendTo(q));
+p.nTHead=b[0];b=q.children("tbody");b.length===0&&(b=h("<tbody/>").appendTo(q));p.nTBody=b[0];b=q.children("tfoot");if(b.length===0&&a.length>0&&(p.oScroll.sX!==""||p.oScroll.sY!==""))b=h("<tfoot/>").appendTo(q);if(b.length===0||b.children().length===0)q.addClass(u.sNoFooter);else if(b.length>0){p.nTFoot=b[0];ea(p.aoFooter,p.nTFoot)}if(g.aaData)for(j=0;j<g.aaData.length;j++)O(p,g.aaData[j]);else(p.bDeferLoading||y(p)=="dom")&&na(p,h(p.nTBody).children("tr"));p.aiDisplay=p.aiDisplayMaster.slice();
+p.bInitialised=true;m===false&&ha(p)};g.bStateSave?(U.bStateSave=!0,z(p,"aoDrawCallback",xa,"state_save"),Jb(p,g,e)):e()}});b=null;return this},x,s,o,u,Za={},Mb=/[\r\n]/g,Aa=/<.*?>/g,Zb=/^\d{2,4}[\.\/\-]\d{1,2}[\.\/\-]\d{1,2}([T ]{1}\d{1,2}[:\.]\d{2}([\.:]\d{2})?)?$/,$b=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),Ya=/[',$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfkɃΞ]/gi,M=function(a){return!a||!0===a||"-"===a?!0:!1},Nb=function(a){var b=parseInt(a,10);return!isNaN(b)&&
+isFinite(a)?b:null},Ob=function(a,b){Za[b]||(Za[b]=RegExp(Qa(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace(Za[b],"."):a},$a=function(a,b,c){var d="string"===typeof a;if(M(a))return!0;b&&d&&(a=Ob(a,b));c&&d&&(a=a.replace(Ya,""));return!isNaN(parseFloat(a))&&isFinite(a)},Pb=function(a,b,c){return M(a)?!0:!(M(a)||"string"===typeof a)?null:$a(a.replace(Aa,""),b,c)?!0:null},D=function(a,b,c){var d=[],e=0,f=a.length;if(c!==k)for(;e<f;e++)a[e]&&a[e][b]&&d.push(a[e][b][c]);else for(;e<
+f;e++)a[e]&&d.push(a[e][b]);return d},ja=function(a,b,c,d){var e=[],f=0,g=b.length;if(d!==k)for(;f<g;f++)a[b[f]][c]&&e.push(a[b[f]][c][d]);else for(;f<g;f++)e.push(a[b[f]][c]);return e},Y=function(a,b){var c=[],d;b===k?(b=0,d=a):(d=b,b=a);for(var e=b;e<d;e++)c.push(e);return c},Qb=function(a){for(var b=[],c=0,d=a.length;c<d;c++)a[c]&&b.push(a[c]);return b},qa=function(a){var b;a:{if(!(2>a.length)){b=a.slice().sort();for(var c=b[0],d=1,e=b.length;d<e;d++){if(b[d]===c){b=!1;break a}c=b[d]}}b=!0}if(b)return a.slice();
+b=[];var e=a.length,f,g=0,d=0;a:for(;d<e;d++){c=a[d];for(f=0;f<g;f++)if(b[f]===c)continue a;b.push(c);g++}return b};n.util={throttle:function(a,b){var c=b!==k?b:200,d,e;return function(){var b=this,g=+new Date,j=arguments;d&&g<d+c?(clearTimeout(e),e=setTimeout(function(){d=k;a.apply(b,j)},c)):(d=g,a.apply(b,j))}},escapeRegex:function(a){return a.replace($b,"\\$1")}};var A=function(a,b,c){a[b]!==k&&(a[c]=a[b])},ca=/\[.*?\]$/,W=/\(\)$/,Qa=n.util.escapeRegex,va=h("<div>")[0],Wb=va.textContent!==k,Yb=
+/<.*?>/g,Oa=n.util.throttle,Rb=[],w=Array.prototype,ac=function(a){var b,c,d=n.settings,e=h.map(d,function(a){return a.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase())return b=h.inArray(a,e),-1!==b?[d[b]]:null;if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?c=h(a):a instanceof h&&(c=a)}else return[];if(c)return c.map(function(){b=h.inArray(this,e);return-1!==b?d[b]:null}).toArray()};s=function(a,b){if(!(this instanceof
+s))return new s(a,b);var c=[],d=function(a){(a=ac(a))&&(c=c.concat(a))};if(h.isArray(a))for(var e=0,f=a.length;e<f;e++)d(a[e]);else d(a);this.context=qa(c);b&&h.merge(this,b);this.selector={rows:null,cols:null,opts:null};s.extend(this,this,Rb)};n.Api=s;h.extend(s.prototype,{any:function(){return 0!==this.count()},concat:w.concat,context:[],count:function(){return this.flatten().length},each:function(a){for(var b=0,c=this.length;b<c;b++)a.call(this,this[b],b,this);return this},eq:function(a){var b=
+this.context;return b.length>a?new s(b[a],this[a]):null},filter:function(a){var b=[];if(w.filter)b=w.filter.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)a.call(this,this[c],c,this)&&b.push(this[c]);return new s(this.context,b)},flatten:function(){var a=[];return new s(this.context,a.concat.apply(a,this.toArray()))},join:w.join,indexOf:w.indexOf||function(a,b){for(var c=b||0,d=this.length;c<d;c++)if(this[c]===a)return c;return-1},iterator:function(a,b,c,d){var e=[],f,g,j,h,m,l=this.context,
+n,o,u=this.selector;"string"===typeof a&&(d=c,c=b,b=a,a=!1);g=0;for(j=l.length;g<j;g++){var r=new s(l[g]);if("table"===b)f=c.call(r,l[g],g),f!==k&&e.push(f);else if("columns"===b||"rows"===b)f=c.call(r,l[g],this[g],g),f!==k&&e.push(f);else if("column"===b||"column-rows"===b||"row"===b||"cell"===b){o=this[g];"column-rows"===b&&(n=Ba(l[g],u.opts));h=0;for(m=o.length;h<m;h++)f=o[h],f="cell"===b?c.call(r,l[g],f.row,f.column,g,h):c.call(r,l[g],f,g,h,n),f!==k&&e.push(f)}}return e.length||d?(a=new s(l,a?
+e.concat.apply([],e):e),b=a.selector,b.rows=u.rows,b.cols=u.cols,b.opts=u.opts,a):this},lastIndexOf:w.lastIndexOf||function(a,b){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,map:function(a){var b=[];if(w.map)b=w.map.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)b.push(a.call(this,this[c],c));return new s(this.context,b)},pluck:function(a){return this.map(function(b){return b[a]})},pop:w.pop,push:w.push,reduce:w.reduce||function(a,b){return hb(this,a,b,0,this.length,
+1)},reduceRight:w.reduceRight||function(a,b){return hb(this,a,b,this.length-1,-1,-1)},reverse:w.reverse,selector:null,shift:w.shift,slice:function(){return new s(this.context,this)},sort:w.sort,splice:w.splice,toArray:function(){return w.slice.call(this)},to$:function(){return h(this)},toJQuery:function(){return h(this)},unique:function(){return new s(this.context,qa(this))},unshift:w.unshift});s.extend=function(a,b,c){if(c.length&&b&&(b instanceof s||b.__dt_wrapper)){var d,e,f,g=function(a,b,c){return function(){var d=
+b.apply(a,arguments);s.extend(d,d,c.methodExt);return d}};d=0;for(e=c.length;d<e;d++)f=c[d],b[f.name]="function"===typeof f.val?g(a,f.val,f):h.isPlainObject(f.val)?{}:f.val,b[f.name].__dt_wrapper=!0,s.extend(a,b[f.name],f.propExt)}};s.register=o=function(a,b){if(h.isArray(a))for(var c=0,d=a.length;c<d;c++)s.register(a[c],b);else for(var e=a.split("."),f=Rb,g,j,c=0,d=e.length;c<d;c++){g=(j=-1!==e[c].indexOf("()"))?e[c].replace("()",""):e[c];var i;a:{i=0;for(var m=f.length;i<m;i++)if(f[i].name===g){i=
+f[i];break a}i=null}i||(i={name:g,val:{},methodExt:[],propExt:[]},f.push(i));c===d-1?i.val=b:f=j?i.methodExt:i.propExt}};s.registerPlural=u=function(a,b,c){s.register(a,c);s.register(b,function(){var a=c.apply(this,arguments);return a===this?this:a instanceof s?a.length?h.isArray(a[0])?new s(a.context,a[0]):a[0]:k:a})};o("tables()",function(a){var b;if(a){b=s;var c=this.context;if("number"===typeof a)a=[c[a]];else var d=h.map(c,function(a){return a.nTable}),a=h(d).filter(a).map(function(){var a=h.inArray(this,
+d);return c[a]}).toArray();b=new b(a)}else b=this;return b});o("table()",function(a){var a=this.tables(a),b=a.context;return b.length?new s(b[0]):a});u("tables().nodes()","table().node()",function(){return this.iterator("table",function(a){return a.nTable},1)});u("tables().body()","table().body()",function(){return this.iterator("table",function(a){return a.nTBody},1)});u("tables().header()","table().header()",function(){return this.iterator("table",function(a){return a.nTHead},1)});u("tables().footer()",
+"table().footer()",function(){return this.iterator("table",function(a){return a.nTFoot},1)});u("tables().containers()","table().container()",function(){return this.iterator("table",function(a){return a.nTableWrapper},1)});o("draw()",function(a){return this.iterator("table",function(b){"page"===a?P(b):("string"===typeof a&&(a="full-hold"===a?!1:!0),T(b,!1===a))})});o("page()",function(a){return a===k?this.page.info().page:this.iterator("table",function(b){Ta(b,a)})});o("page.info()",function(){if(0===
+this.context.length)return k;var a=this.context[0],b=a._iDisplayStart,c=a.oFeatures.bPaginate?a._iDisplayLength:-1,d=a.fnRecordsDisplay(),e=-1===c;return{page:e?0:Math.floor(b/c),pages:e?1:Math.ceil(d/c),start:b,end:a.fnDisplayEnd(),length:c,recordsTotal:a.fnRecordsTotal(),recordsDisplay:d,serverSide:"ssp"===y(a)}});o("page.len()",function(a){return a===k?0!==this.context.length?this.context[0]._iDisplayLength:k:this.iterator("table",function(b){Ra(b,a)})});var Sb=function(a,b,c){if(c){var d=new s(a);
+d.one("draw",function(){c(d.ajax.json())})}if("ssp"==y(a))T(a,b);else{C(a,!0);var e=a.jqXHR;e&&4!==e.readyState&&e.abort();sa(a,[],function(c){oa(a);for(var c=ta(a,c),d=0,e=c.length;d<e;d++)O(a,c[d]);T(a,b);C(a,!1)})}};o("ajax.json()",function(){var a=this.context;if(0<a.length)return a[0].json});o("ajax.params()",function(){var a=this.context;if(0<a.length)return a[0].oAjaxData});o("ajax.reload()",function(a,b){return this.iterator("table",function(c){Sb(c,!1===b,a)})});o("ajax.url()",function(a){var b=
+this.context;if(a===k){if(0===b.length)return k;b=b[0];return b.ajax?h.isPlainObject(b.ajax)?b.ajax.url:b.ajax:b.sAjaxSource}return this.iterator("table",function(b){h.isPlainObject(b.ajax)?b.ajax.url=a:b.ajax=a})});o("ajax.url().load()",function(a,b){return this.iterator("table",function(c){Sb(c,!1===b,a)})});var ab=function(a,b,c,d,e){var f=[],g,j,i,m,l,n;i=typeof b;if(!b||"string"===i||"function"===i||b.length===k)b=[b];i=0;for(m=b.length;i<m;i++){j=b[i]&&b[i].split&&!b[i].match(/[\[\(:]/)?b[i].split(","):
+[b[i]];l=0;for(n=j.length;l<n;l++)(g=c("string"===typeof j[l]?h.trim(j[l]):j[l]))&&g.length&&(f=f.concat(g))}a=x.selector[a];if(a.length){i=0;for(m=a.length;i<m;i++)f=a[i](d,e,f)}return qa(f)},bb=function(a){a||(a={});a.filter&&a.search===k&&(a.search=a.filter);return h.extend({search:"none",order:"current",page:"all"},a)},cb=function(a){for(var b=0,c=a.length;b<c;b++)if(0<a[b].length)return a[0]=a[b],a[0].length=1,a.length=1,a.context=[a.context[b]],a;a.length=0;return a},Ba=function(a,b){var c,
+d,e,f=[],g=a.aiDisplay;e=a.aiDisplayMaster;var j=b.search;c=b.order;d=b.page;if("ssp"==y(a))return"removed"===j?[]:Y(0,e.length);if("current"==d){c=a._iDisplayStart;for(d=a.fnDisplayEnd();c<d;c++)f.push(g[c])}else if("current"==c||"applied"==c)if("none"==j)f=e.slice();else if("applied"==j)f=g.slice();else{if("removed"==j){var i={};c=0;for(d=g.length;c<d;c++)i[g[c]]=null;f=h.map(e,function(a){return!i.hasOwnProperty(a)?a:null})}}else if("index"==c||"original"==c){c=0;for(d=a.aoData.length;c<d;c++)"none"==
+j?f.push(c):(e=h.inArray(c,g),(-1===e&&"removed"==j||0<=e&&"applied"==j)&&f.push(c))}return f};o("rows()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=bb(b),c=this.iterator("table",function(c){var e=b,f;return ab("row",a,function(a){var b=Nb(a),i=c.aoData;if(b!==null&&!e)return[b];f||(f=Ba(c,e));if(b!==null&&h.inArray(b,f)!==-1)return[b];if(a===null||a===k||a==="")return f;if(typeof a==="function")return h.map(f,function(b){var c=i[b];return a(b,c._aData,c.nTr)?b:null});if(a.nodeName){var b=
+a._DT_RowIndex,m=a._DT_CellIndex;if(b!==k)return i[b]&&i[b].nTr===a?[b]:[];if(m)return i[m.row]&&i[m.row].nTr===a?[m.row]:[];b=h(a).closest("*[data-dt-row]");return b.length?[b.data("dt-row")]:[]}if(typeof a==="string"&&a.charAt(0)==="#"){b=c.aIds[a.replace(/^#/,"")];if(b!==k)return[b.idx]}b=Qb(ja(c.aoData,f,"nTr"));return h(b).filter(a).map(function(){return this._DT_RowIndex}).toArray()},c,e)},1);c.selector.rows=a;c.selector.opts=b;return c});o("rows().nodes()",function(){return this.iterator("row",
+function(a,b){return a.aoData[b].nTr||k},1)});o("rows().data()",function(){return this.iterator(!0,"rows",function(a,b){return ja(a.aoData,b,"_aData")},1)});u("rows().cache()","row().cache()",function(a){return this.iterator("row",function(b,c){var d=b.aoData[c];return"search"===a?d._aFilterData:d._aSortData},1)});u("rows().invalidate()","row().invalidate()",function(a){return this.iterator("row",function(b,c){da(b,c,a)})});u("rows().indexes()","row().index()",function(){return this.iterator("row",
+function(a,b){return b},1)});u("rows().ids()","row().id()",function(a){for(var b=[],c=this.context,d=0,e=c.length;d<e;d++)for(var f=0,g=this[d].length;f<g;f++){var h=c[d].rowIdFn(c[d].aoData[this[d][f]]._aData);b.push((!0===a?"#":"")+h)}return new s(c,b)});u("rows().remove()","row().remove()",function(){var a=this;this.iterator("row",function(b,c,d){var e=b.aoData,f=e[c],g,h,i,m,l;e.splice(c,1);g=0;for(h=e.length;g<h;g++)if(i=e[g],l=i.anCells,null!==i.nTr&&(i.nTr._DT_RowIndex=g),null!==l){i=0;for(m=
+l.length;i<m;i++)l[i]._DT_CellIndex.row=g}pa(b.aiDisplayMaster,c);pa(b.aiDisplay,c);pa(a[d],c,!1);0<b._iRecordsDisplay&&b._iRecordsDisplay--;Sa(b);c=b.rowIdFn(f._aData);c!==k&&delete b.aIds[c]});this.iterator("table",function(a){for(var c=0,d=a.aoData.length;c<d;c++)a.aoData[c].idx=c});return this});o("rows.add()",function(a){var b=this.iterator("table",function(b){var c,f,g,h=[];f=0;for(g=a.length;f<g;f++)c=a[f],c.nodeName&&"TR"===c.nodeName.toUpperCase()?h.push(na(b,c)[0]):h.push(O(b,c));return h},
+1),c=this.rows(-1);c.pop();h.merge(c,b);return c});o("row()",function(a,b){return cb(this.rows(a,b))});o("row().data()",function(a){var b=this.context;if(a===k)return b.length&&this.length?b[0].aoData[this[0]]._aData:k;var c=b[0].aoData[this[0]];c._aData=a;h.isArray(a)&&c.nTr.id&&N(b[0].rowId)(a,c.nTr.id);da(b[0],this[0],"data");return this});o("row().node()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]].nTr||null:null});o("row.add()",function(a){a instanceof h&&
+a.length&&(a=a[0]);var b=this.iterator("table",function(b){return a.nodeName&&"TR"===a.nodeName.toUpperCase()?na(b,a)[0]:O(b,a)});return this.row(b[0])});var db=function(a,b){var c=a.context;if(c.length&&(c=c[0].aoData[b!==k?b:a[0]])&&c._details)c._details.remove(),c._detailsShow=k,c._details=k},Tb=function(a,b){var c=a.context;if(c.length&&a.length){var d=c[0].aoData[a[0]];if(d._details){(d._detailsShow=b)?d._details.insertAfter(d.nTr):d._details.detach();var e=c[0],f=new s(e),g=e.aoData;f.off("draw.dt.DT_details column-visibility.dt.DT_details destroy.dt.DT_details");
+0<D(g,"_details").length&&(f.on("draw.dt.DT_details",function(a,b){e===b&&f.rows({page:"current"}).eq(0).each(function(a){a=g[a];a._detailsShow&&a._details.insertAfter(a.nTr)})}),f.on("column-visibility.dt.DT_details",function(a,b){if(e===b)for(var c,d=V(b),f=0,h=g.length;f<h;f++)c=g[f],c._details&&c._details.children("td[colspan]").attr("colspan",d)}),f.on("destroy.dt.DT_details",function(a,b){if(e===b)for(var c=0,d=g.length;c<d;c++)g[c]._details&&db(f,c)}))}}};o("row().child()",function(a,b){var c=
+this.context;if(a===k)return c.length&&this.length?c[0].aoData[this[0]]._details:k;if(!0===a)this.child.show();else if(!1===a)db(this);else if(c.length&&this.length){var d=c[0],c=c[0].aoData[this[0]],e=[],f=function(a,b){if(h.isArray(a)||a instanceof h)for(var c=0,k=a.length;c<k;c++)f(a[c],b);else a.nodeName&&"tr"===a.nodeName.toLowerCase()?e.push(a):(c=h("<tr><td/></tr>").addClass(b),h("td",c).addClass(b).html(a)[0].colSpan=V(d),e.push(c[0]))};f(a,b);c._details&&c._details.detach();c._details=h(e);
+c._detailsShow&&c._details.insertAfter(c.nTr)}return this});o(["row().child.show()","row().child().show()"],function(){Tb(this,!0);return this});o(["row().child.hide()","row().child().hide()"],function(){Tb(this,!1);return this});o(["row().child.remove()","row().child().remove()"],function(){db(this);return this});o("row().child.isShown()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var bc=/^([^:]+):(name|visIdx|visible)$/,Ub=function(a,b,
+c,d,e){for(var c=[],d=0,f=e.length;d<f;d++)c.push(B(a,e[d],b));return c};o("columns()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=bb(b),c=this.iterator("table",function(c){var e=a,f=b,g=c.aoColumns,j=D(g,"sName"),i=D(g,"nTh");return ab("column",e,function(a){var b=Nb(a);if(a==="")return Y(g.length);if(b!==null)return[b>=0?b:g.length+b];if(typeof a==="function"){var e=Ba(c,f);return h.map(g,function(b,f){return a(f,Ub(c,f,0,0,e),i[f])?f:null})}var k=typeof a==="string"?a.match(bc):
+"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var n=h.map(g,function(a,b){return a.bVisible?b:null});return[n[n.length+b]]}return[aa(c,b)];case "name":return h.map(j,function(a,b){return a===k[1]?b:null});default:return[]}if(a.nodeName&&a._DT_CellIndex)return[a._DT_CellIndex.column];b=h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray();if(b.length||!a.nodeName)return b;b=h(a).closest("*[data-dt-column]");return b.length?[b.data("dt-column")]:[]},c,f)},
+1);c.selector.cols=a;c.selector.opts=b;return c});u("columns().header()","column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});u("columns().footer()","column().footer()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});u("columns().data()","column().data()",function(){return this.iterator("column-rows",Ub,1)});u("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData},
+1)});u("columns().cache()","column().cache()",function(a){return this.iterator("column-rows",function(b,c,d,e,f){return ja(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});u("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,d,e){return ja(a.aoData,e,"anCells",b)},1)});u("columns().visible()","column().visible()",function(a,b){var c=this.iterator("column",function(b,c){if(a===k)return b.aoColumns[c].bVisible;var f=b.aoColumns,g=f[c],j=b.aoData,
+i,m,l;if(a!==k&&g.bVisible!==a){if(a){var n=h.inArray(!0,D(f,"bVisible"),c+1);i=0;for(m=j.length;i<m;i++)l=j[i].nTr,f=j[i].anCells,l&&l.insertBefore(f[c],f[n]||null)}else h(D(b.aoData,"anCells",c)).detach();g.bVisible=a;fa(b,b.aoHeader);fa(b,b.aoFooter);b.aiDisplay.length||h(b.nTBody).find("td[colspan]").attr("colspan",V(b));xa(b)}});a!==k&&(this.iterator("column",function(c,e){r(c,null,"column-visibility",[c,e,a,b])}),(b===k||b)&&this.columns.adjust());return c});u("columns().indexes()","column().index()",
+function(a){return this.iterator("column",function(b,c){return"visible"===a?ba(b,c):c},1)});o("columns.adjust()",function(){return this.iterator("table",function(a){$(a)},1)});o("column.index()",function(a,b){if(0!==this.context.length){var c=this.context[0];if("fromVisible"===a||"toData"===a)return aa(c,b);if("fromData"===a||"toVisible"===a)return ba(c,b)}});o("column()",function(a,b){return cb(this.columns(a,b))});o("cells()",function(a,b,c){h.isPlainObject(a)&&(a.row===k?(c=a,a=null):(c=b,b=null));
+h.isPlainObject(b)&&(c=b,b=null);if(null===b||b===k)return this.iterator("table",function(b){var d=a,e=bb(c),f=b.aoData,g=Ba(b,e),j=Qb(ja(f,g,"anCells")),i=h([].concat.apply([],j)),l,m=b.aoColumns.length,n,o,u,s,r,v;return ab("cell",d,function(a){var c=typeof a==="function";if(a===null||a===k||c){n=[];o=0;for(u=g.length;o<u;o++){l=g[o];for(s=0;s<m;s++){r={row:l,column:s};if(c){v=f[l];a(r,B(b,l,s),v.anCells?v.anCells[s]:null)&&n.push(r)}else n.push(r)}}return n}if(h.isPlainObject(a))return a.column!==
+k&&a.row!==k&&h.inArray(a.row,g)!==-1?[a]:[];c=i.filter(a).map(function(a,b){return{row:b._DT_CellIndex.row,column:b._DT_CellIndex.column}}).toArray();if(c.length||!a.nodeName)return c;v=h(a).closest("*[data-dt-row]");return v.length?[{row:v.data("dt-row"),column:v.data("dt-column")}]:[]},b,e)});var d=this.columns(b),e=this.rows(a),f,g,j,i,m;this.iterator("table",function(a,b){f=[];g=0;for(j=e[b].length;g<j;g++){i=0;for(m=d[b].length;i<m;i++)f.push({row:e[b][g],column:d[b][i]})}},1);var l=this.cells(f,
+c);h.extend(l.selector,{cols:b,rows:a,opts:c});return l});u("cells().nodes()","cell().node()",function(){return this.iterator("cell",function(a,b,c){return(a=a.aoData[b])&&a.anCells?a.anCells[c]:k},1)});o("cells().data()",function(){return this.iterator("cell",function(a,b,c){return B(a,b,c)},1)});u("cells().cache()","cell().cache()",function(a){a="search"===a?"_aFilterData":"_aSortData";return this.iterator("cell",function(b,c,d){return b.aoData[c][a][d]},1)});u("cells().render()","cell().render()",
+function(a){return this.iterator("cell",function(b,c,d){return B(b,c,d,a)},1)});u("cells().indexes()","cell().index()",function(){return this.iterator("cell",function(a,b,c){return{row:b,column:c,columnVisible:ba(a,c)}},1)});u("cells().invalidate()","cell().invalidate()",function(a){return this.iterator("cell",function(b,c,d){da(b,c,a,d)})});o("cell()",function(a,b,c){return cb(this.cells(a,b,c))});o("cell().data()",function(a){var b=this.context,c=this[0];if(a===k)return b.length&&c.length?B(b[0],
+c[0].row,c[0].column):k;jb(b[0],c[0].row,c[0].column,a);da(b[0],c[0].row,"data",c[0].column);return this});o("order()",function(a,b){var c=this.context;if(a===k)return 0!==c.length?c[0].aaSorting:k;"number"===typeof a?a=[[a,b]]:a.length&&!h.isArray(a[0])&&(a=Array.prototype.slice.call(arguments));return this.iterator("table",function(b){b.aaSorting=a.slice()})});o("order.listener()",function(a,b,c){return this.iterator("table",function(d){Ma(d,a,b,c)})});o("order.fixed()",function(a){if(!a){var b=
+this.context,b=b.length?b[0].aaSortingFixed:k;return h.isArray(b)?{pre:b}:b}return this.iterator("table",function(b){b.aaSortingFixed=h.extend(!0,{},a)})});o(["columns().order()","column().order()"],function(a){var b=this;return this.iterator("table",function(c,d){var e=[];h.each(b[d],function(b,c){e.push([c,a])});c.aaSorting=e})});o("search()",function(a,b,c,d){var e=this.context;return a===k?0!==e.length?e[0].oPreviousSearch.sSearch:k:this.iterator("table",function(e){e.oFeatures.bFilter&&ga(e,
+h.extend({},e.oPreviousSearch,{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),1)})});u("columns().search()","column().search()",function(a,b,c,d){return this.iterator("column",function(e,f){var g=e.aoPreSearchCols;if(a===k)return g[f].sSearch;e.oFeatures.bFilter&&(h.extend(g[f],{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),ga(e,e.oPreviousSearch,1))})});o("state()",function(){return this.context.length?this.context[0].oSavedState:
+null});o("state.clear()",function(){return this.iterator("table",function(a){a.fnStateSaveCallback.call(a.oInstance,a,{})})});o("state.loaded()",function(){return this.context.length?this.context[0].oLoadedState:null});o("state.save()",function(){return this.iterator("table",function(a){xa(a)})});n.versionCheck=n.fnVersionCheck=function(a){for(var b=n.version.split("."),a=a.split("."),c,d,e=0,f=a.length;e<f;e++)if(c=parseInt(b[e],10)||0,d=parseInt(a[e],10)||0,c!==d)return c>d;return!0};n.isDataTable=
+n.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;if(a instanceof n.Api)return!0;h.each(n.settings,function(a,e){var f=e.nScrollHead?h("table",e.nScrollHead)[0]:null,g=e.nScrollFoot?h("table",e.nScrollFoot)[0]:null;if(e.nTable===b||f===b||g===b)c=!0});return c};n.tables=n.fnTables=function(a){var b=!1;h.isPlainObject(a)&&(b=a.api,a=a.visible);var c=h.map(n.settings,function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable});return b?new s(c):c};n.camelToHungarian=J;o("$()",function(a,b){var c=
+this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a,b){o(b+"()",function(){var a=Array.prototype.slice.call(arguments);a[0]=h.map(a[0].split(/\s/),function(a){return!a.match(/\.dt\b/)?a+".dt":a}).join(" ");var d=h(this.tables().nodes());d[b].apply(d,a);return this})});o("clear()",function(){return this.iterator("table",function(a){oa(a)})});o("settings()",function(){return new s(this.context,this.context)});o("init()",function(){var a=
+this.context;return a.length?a[0].oInit:null});o("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});o("destroy()",function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,d=b.oClasses,e=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(e),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),o;b.bDestroying=!0;r(b,"aoDestroyCallback","destroy",[b]);a||(new s(b)).columns().visible(!0);k.off(".DT").find(":not(tbody *)").off(".DT");
+h(E).off(".DT-"+b.sInstance);e!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&e!=j.parentNode&&(i.children("tfoot").detach(),i.append(j));b.aaSorting=[];b.aaSortingFixed=[];wa(b);h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(d.sSortable+" "+d.sSortableAsc+" "+d.sSortableDesc+" "+d.sSortableNone);f.children().detach();f.append(l);g=a?"remove":"detach";i[g]();k[g]();!a&&c&&(c.insertBefore(e,b.nTableReinsertBefore),i.css("width",b.sDestroyWidth).removeClass(d.sTable),
+(o=b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%o])}));c=h.inArray(b,n.settings);-1!==c&&n.settings.splice(c,1)})});h.each(["column","row","cell"],function(a,b){o(b+"s().every()",function(a){var d=this.selector.opts,e=this;return this.iterator(b,function(f,g,h,i,m){a.call(e[b](g,"cell"===b?h:d,"cell"===b?d:k),g,h,i,m)})})});o("i18n()",function(a,b,c){var d=this.context[0],a=S(a)(d.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]:
+a._);return a.replace("%d",c)});n.version="1.10.18";n.settings=[];n.models={};n.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};n.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null,idx:-1};n.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,
+sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};n.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,
+bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+
+a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},
+oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},
+n.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"};Z(n.defaults);n.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};
+Z(n.defaults.column);n.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[],
+aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",
+iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==y(this)?1*this._iRecordsTotal:
+this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==y(this)?1*this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var a=this._iDisplayLength,b=this._iDisplayStart,c=b+a,d=this.aiDisplay.length,e=this.oFeatures,f=e.bPaginate;return e.bServerSide?!1===f||-1===a?b+d:Math.min(b+a,this._iRecordsDisplay):!f||c>d||-1===a?d:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null};n.ext=x={buttons:{},
+classes:{},builder:"-source-",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:n.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:n.version};h.extend(x,{afnFiltering:x.search,aTypes:x.type.detect,ofnSearch:x.type.search,oSort:x.type.order,afnSortData:x.order,aoFeatures:x.feature,oApi:x.internal,oStdClasses:x.classes,oPagination:x.pager});
+h.extend(n.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",
+sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",
+sJUIHeader:"",sJUIFooter:""});var Kb=n.ext.pager;h.extend(Kb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},numbers:function(a,b){return[ia(a,b)]},simple_numbers:function(a,b){return["previous",ia(a,b),"next"]},full_numbers:function(a,b){return["first","previous",ia(a,b),"next","last"]},first_last_numbers:function(a,b){return["first",ia(a,b),"last"]},_numbers:ia,numbers_length:7});h.extend(!0,n.ext.renderer,{pageButton:{_:function(a,b,c,d,e,
+f){var g=a.oClasses,j=a.oLanguage.oPaginate,i=a.oLanguage.oAria.paginate||{},m,l,n=0,o=function(b,d){var k,s,u,r,v=function(b){Ta(a,b.data.action,true)};k=0;for(s=d.length;k<s;k++){r=d[k];if(h.isArray(r)){u=h("<"+(r.DT_el||"div")+"/>").appendTo(b);o(u,r)}else{m=null;l="";switch(r){case "ellipsis":b.append('<span class="ellipsis">…</span>');break;case "first":m=j.sFirst;l=r+(e>0?"":" "+g.sPageButtonDisabled);break;case "previous":m=j.sPrevious;l=r+(e>0?"":" "+g.sPageButtonDisabled);break;case "next":m=
+j.sNext;l=r+(e<f-1?"":" "+g.sPageButtonDisabled);break;case "last":m=j.sLast;l=r+(e<f-1?"":" "+g.sPageButtonDisabled);break;default:m=r+1;l=e===r?g.sPageButtonActive:""}if(m!==null){u=h("<a>",{"class":g.sPageButton+" "+l,"aria-controls":a.sTableId,"aria-label":i[r],"data-dt-idx":n,tabindex:a.iTabIndex,id:c===0&&typeof r==="string"?a.sTableId+"_"+r:null}).html(m).appendTo(b);Wa(u,{action:r},v);n++}}}},s;try{s=h(b).find(H.activeElement).data("dt-idx")}catch(u){}o(h(b).empty(),d);s!==k&&h(b).find("[data-dt-idx="+
+s+"]").focus()}}});h.extend(n.ext.type.detect,[function(a,b){var c=b.oLanguage.sDecimal;return $a(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&!Zb.test(a))return null;var b=Date.parse(a);return null!==b&&!isNaN(b)||M(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return $a(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Pb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Pb(a,c,!0)?"html-num-fmt"+c:null},function(a){return M(a)||
+"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);h.extend(n.ext.type.search,{html:function(a){return M(a)?a:"string"===typeof a?a.replace(Mb," ").replace(Aa,""):""},string:function(a){return M(a)?a:"string"===typeof a?a.replace(Mb," "):a}});var za=function(a,b,c,d){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Ob(a,b));a.replace&&(c&&(a=a.replace(c,"")),d&&(a=a.replace(d,"")));return 1*a};h.extend(x.type.order,{"date-pre":function(a){a=Date.parse(a);return isNaN(a)?-Infinity:a},"html-pre":function(a){return M(a)?
+"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return M(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return a<b?-1:a>b?1:0},"string-desc":function(a,b){return a<b?1:a>b?-1:0}});Da("");h.extend(!0,n.ext.renderer,{header:{_:function(a,b,c,d){h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(c.sSortingClass+" "+d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:
+c.sSortingClass)}})},jqueryui:function(a,b,c,d){h("<div/>").addClass(d.sSortJUIWrapper).append(b.contents()).append(h("<span/>").addClass(d.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass);b.find("span."+d.sSortIcon).removeClass(d.sSortJUIAsc+" "+d.sSortJUIDesc+" "+d.sSortJUI+" "+d.sSortJUIAscAllowed+" "+d.sSortJUIDescAllowed).addClass(h[e]==
+"asc"?d.sSortJUIAsc:h[e]=="desc"?d.sSortJUIDesc:c.sSortingClassJUI)}})}}});var Vb=function(a){return"string"===typeof a?a.replace(/</g,"<").replace(/>/g,">").replace(/"/g,"""):a};n.render={number:function(a,b,c,d,e){return{display:function(f){if("number"!==typeof f&&"string"!==typeof f)return f;var g=0>f?"-":"",h=parseFloat(f);if(isNaN(h))return Vb(f);h=h.toFixed(c);f=Math.abs(h);h=parseInt(f,10);f=c?b+(f-h).toFixed(c).substring(2):"";return g+(d||"")+h.toString().replace(/\B(?=(\d{3})+(?!\d))/g,
+a)+f+(e||"")}}},text:function(){return{display:Vb}}};h.extend(n.ext.internal,{_fnExternApiFunc:Lb,_fnBuildAjax:sa,_fnAjaxUpdate:lb,_fnAjaxParameters:ub,_fnAjaxUpdateDraw:vb,_fnAjaxDataSrc:ta,_fnAddColumn:Ea,_fnColumnOptions:ka,_fnAdjustColumnSizing:$,_fnVisibleToColumnIndex:aa,_fnColumnIndexToVisible:ba,_fnVisbleColumns:V,_fnGetColumns:ma,_fnColumnTypes:Ga,_fnApplyColumnDefs:ib,_fnHungarianMap:Z,_fnCamelToHungarian:J,_fnLanguageCompat:Ca,_fnBrowserDetect:gb,_fnAddData:O,_fnAddTr:na,_fnNodeToDataIndex:function(a,
+b){return b._DT_RowIndex!==k?b._DT_RowIndex:null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:B,_fnSetCellData:jb,_fnSplitObjNotation:Ja,_fnGetObjectDataFn:S,_fnSetObjectDataFn:N,_fnGetDataMaster:Ka,_fnClearTable:oa,_fnDeleteIndex:pa,_fnInvalidate:da,_fnGetRowElements:Ia,_fnCreateTr:Ha,_fnBuildHead:kb,_fnDrawHead:fa,_fnDraw:P,_fnReDraw:T,_fnAddOptionsHtml:nb,_fnDetectHeader:ea,_fnGetUniqueThs:ra,_fnFeatureHtmlFilter:pb,_fnFilterComplete:ga,_fnFilterCustom:yb,
+_fnFilterColumn:xb,_fnFilter:wb,_fnFilterCreateSearch:Pa,_fnEscapeRegex:Qa,_fnFilterData:zb,_fnFeatureHtmlInfo:sb,_fnUpdateInfo:Cb,_fnInfoMacros:Db,_fnInitialise:ha,_fnInitComplete:ua,_fnLengthChange:Ra,_fnFeatureHtmlLength:ob,_fnFeatureHtmlPaginate:tb,_fnPageChange:Ta,_fnFeatureHtmlProcessing:qb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:rb,_fnScrollDraw:la,_fnApplyToChildren:I,_fnCalculateColumnWidths:Fa,_fnThrottle:Oa,_fnConvertToWidth:Eb,_fnGetWidestNode:Fb,_fnGetMaxLenString:Gb,_fnStringToCss:v,
+_fnSortFlatten:X,_fnSort:mb,_fnSortAria:Ib,_fnSortListener:Va,_fnSortAttachListener:Ma,_fnSortingClasses:wa,_fnSortData:Hb,_fnSaveState:xa,_fnLoadState:Jb,_fnSettingsFromNode:ya,_fnLog:K,_fnMap:F,_fnBindAction:Wa,_fnCallbackReg:z,_fnCallbackFire:r,_fnLengthOverflow:Sa,_fnRenderer:Na,_fnDataSource:y,_fnRowAttributes:La,_fnExtend:Xa,_fnCalculateEnd:function(){}});h.fn.dataTable=n;n.$=h;h.fn.dataTableSettings=n.settings;h.fn.dataTableExt=n.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()};
+h.each(n,function(a,b){h.fn.DataTable[a]=b});return h.fn.dataTable});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/demo_page.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/demo_page.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/demo_page.css
deleted file mode 100644
index b60ee7d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/css/demo_page.css
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * General page setup
- */
-#dt_example {
- font: 80%/1.45em "Lucida Grande", Verdana, Arial, Helvetica, sans-serif;
- margin: 0;
- padding: 0;
- color: #333;
- background-color: #fff;
-}
-
-
-#dt_example #container {
- width: 800px;
- margin: 30px auto;
- padding: 0;
-}
-
-
-#dt_example #footer {
- margin: 50px auto 0 auto;
- padding: 0;
-}
-
-#dt_example #demo {
- margin: 30px auto 0 auto;
-}
-
-#dt_example .demo_jui {
- margin: 30px auto 0 auto;
-}
-
-#dt_example .big {
- font-size: 1.3em;
- font-weight: bold;
- line-height: 1.6em;
- color: #4E6CA3;
-}
-
-#dt_example .spacer {
- height: 20px;
- clear: both;
-}
-
-#dt_example .clear {
- clear: both;
-}
-
-#dt_example pre {
- padding: 15px;
- background-color: #F5F5F5;
- border: 1px solid #CCCCCC;
-}
-
-#dt_example h1 {
- margin-top: 2em;
- font-size: 1.3em;
- font-weight: normal;
- line-height: 1.6em;
- color: #4E6CA3;
- border-bottom: 1px solid #B0BED9;
- clear: both;
-}
-
-#dt_example h2 {
- font-size: 1.2em;
- font-weight: normal;
- line-height: 1.6em;
- color: #4E6CA3;
- clear: both;
-}
-
-#dt_example a {
- color: #0063DC;
- text-decoration: none;
-}
-
-#dt_example a:hover {
- text-decoration: underline;
-}
-
-#dt_example ul {
- color: #4E6CA3;
-}
-
-.css_right {
- float: right;
-}
-
-.css_left {
- float: left;
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-8893. [AMRMProxy] Fix thread
leak in AMRMClientRelayer and UAM client. Contributed by Botong Huang.
Posted by ae...@apache.org.
YARN-8893. [AMRMProxy] Fix thread leak in AMRMClientRelayer and UAM client. Contributed by Botong Huang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/989715ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/989715ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/989715ec
Branch: refs/heads/HDDS-4
Commit: 989715ec5066c6ac7868e25ad9234dc64723e61e
Parents: aed836e
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 2 15:30:08 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Nov 2 15:30:08 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/AMRMClientRelayer.java | 55 +++++---------------
.../yarn/server/uam/UnmanagedAMPoolManager.java | 28 ++++++++++
.../server/uam/UnmanagedApplicationManager.java | 28 +++++++---
.../yarn/server/MockResourceManagerFacade.java | 5 +-
.../yarn/server/TestAMRMClientRelayer.java | 10 ++--
.../metrics/TestAMRMClientRelayerMetrics.java | 6 ---
.../uam/TestUnmanagedApplicationManager.java | 27 +++++++++-
.../amrmproxy/FederationInterceptor.java | 18 +++----
.../amrmproxy/TestAMRMProxyService.java | 1 -
9 files changed, 102 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989715ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
index dc66868..ac43b12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
@@ -27,9 +27,8 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -47,8 +46,6 @@ import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.client.AMRMClientUtils;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -66,8 +63,7 @@ import com.google.common.annotations.VisibleForTesting;
* pending requests similar to AMRMClient, and handles RM re-sync automatically
* without propagate the re-sync exception back to AMRMClient.
*/
-public class AMRMClientRelayer extends AbstractService
- implements ApplicationMasterProtocol {
+public class AMRMClientRelayer implements ApplicationMasterProtocol {
private static final Logger LOG =
LoggerFactory.getLogger(AMRMClientRelayer.class);
@@ -136,51 +132,16 @@ public class AMRMClientRelayer extends AbstractService
private AMRMClientRelayerMetrics metrics;
- public AMRMClientRelayer() {
- super(AMRMClientRelayer.class.getName());
+ public AMRMClientRelayer(ApplicationMasterProtocol rmClient,
+ ApplicationId appId, String rmId) {
this.resetResponseId = -1;
this.metrics = AMRMClientRelayerMetrics.getInstance();
- this.rmClient = null;
- this.appId = null;
this.rmId = "";
- }
-
- public AMRMClientRelayer(ApplicationMasterProtocol rmClient,
- ApplicationId appId, String rmId) {
- this();
this.rmClient = rmClient;
this.appId = appId;
this.rmId = rmId;
}
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- final YarnConfiguration conf = new YarnConfiguration(getConfig());
- try {
- if (this.rmClient == null) {
- this.rmClient =
- ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
- }
- } catch (IOException e) {
- throw new YarnRuntimeException(e);
- }
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (this.rmClient != null) {
- RPC.stopProxy(this.rmClient);
- }
- shutdown();
- super.serviceStop();
- }
-
public void setAMRegistrationRequest(
RegisterApplicationMasterRequest registerRequest) {
this.amRegistrationRequest = registerRequest;
@@ -231,6 +192,14 @@ public class AMRMClientRelayer extends AbstractService
.decrClientPending(rmId, req.getContainerUpdateType(), 1);
}
}
+
+ if (this.rmClient != null) {
+ try {
+ RPC.stopProxy(this.rmClient);
+ this.rmClient = null;
+ } catch (HadoopIllegalArgumentException e) {
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989715ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
index d5a0168..7072030 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
@@ -372,6 +372,34 @@ public class UnmanagedAMPoolManager extends AbstractService {
}
/**
+ * Shutdown an UAM client without killing it in YarnRM.
+ *
+ * @param uamId uam Id
+ * @throws YarnException if fails
+ */
+ public void shutDownConnections(String uamId)
+ throws YarnException {
+ if (!this.unmanagedAppMasterMap.containsKey(uamId)) {
+ throw new YarnException("UAM " + uamId + " does not exist");
+ }
+ LOG.info(
+ "Shutting down UAM id {} for application {} without killing the UAM",
+ uamId, this.appIdMap.get(uamId));
+ this.unmanagedAppMasterMap.remove(uamId).shutDownConnections();
+ }
+
+ /**
+ * Shutdown all UAM clients without killing them in YarnRM.
+ *
+ * @throws YarnException if fails
+ */
+ public void shutDownConnections() throws YarnException {
+ for (String uamId : this.unmanagedAppMasterMap.keySet()) {
+ shutDownConnections(uamId);
+ }
+ }
+
+ /**
* Get the id of all running UAMs.
*
* @return uamId set
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989715ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
index 134df1d..cc1d21f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
@@ -255,9 +255,6 @@ public class UnmanagedApplicationManager {
public FinishApplicationMasterResponse finishApplicationMaster(
FinishApplicationMasterRequest request)
throws YarnException, IOException {
-
- this.heartbeatHandler.shutdown();
-
if (this.userUgi == null) {
if (this.connectionInitiated) {
// This is possible if the async launchUAM is still
@@ -270,7 +267,12 @@ public class UnmanagedApplicationManager {
+ "be called before createAndRegister");
}
}
- return this.rmProxyRelayer.finishApplicationMaster(request);
+ FinishApplicationMasterResponse response =
+ this.rmProxyRelayer.finishApplicationMaster(request);
+ if (response.getIsUnregistered()) {
+ shutDownConnections();
+ }
+ return response;
}
/**
@@ -282,11 +284,10 @@ public class UnmanagedApplicationManager {
*/
public KillApplicationResponse forceKillApplication()
throws IOException, YarnException {
+ shutDownConnections();
+
KillApplicationRequest request =
KillApplicationRequest.newInstance(this.applicationId);
-
- this.heartbeatHandler.shutdown();
-
if (this.rmClient == null) {
this.rmClient = createRMProxy(ApplicationClientProtocol.class, this.conf,
UserGroupInformation.createRemoteUser(this.submitter), null);
@@ -324,6 +325,14 @@ public class UnmanagedApplicationManager {
}
/**
+ * Shutdown this UAM client, without killing the UAM in the YarnRM side.
+ */
+ public void shutDownConnections() {
+ this.heartbeatHandler.shutdown();
+ this.rmProxyRelayer.shutdown();
+ }
+
+ /**
* Returns the application id of the UAM.
*
* @return application id of the UAM
@@ -532,4 +541,9 @@ public class UnmanagedApplicationManager {
protected void drainHeartbeatThread() {
this.heartbeatHandler.drainHeartbeatThread();
}
+
+ @VisibleForTesting
+ protected boolean isHeartbeatThreadAlive() {
+ return this.heartbeatHandler.isAlive();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989715ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
index 60c2ac9..16ba903 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
@@ -120,7 +120,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
@@ -338,9 +337,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
applicationContainerIdMap.remove(appId);
}
- return FinishApplicationMasterResponse.newInstance(
- request.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED
- ? true : false);
+ return FinishApplicationMasterResponse.newInstance(true);
}
protected ApplicationId getApplicationId(int id) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989715ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
index fa46960..46570a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestExceptio
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.scheduler.ResourceRequestSet;
import org.apache.hadoop.yarn.util.Records;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -155,16 +156,17 @@ public class TestAMRMClientRelayer {
this.mockAMS = new MockApplicationMasterService();
this.relayer = new AMRMClientRelayer(this.mockAMS, null, "TEST");
-
- this.relayer.init(conf);
- this.relayer.start();
-
this.relayer.registerApplicationMaster(
RegisterApplicationMasterRequest.newInstance("", 0, ""));
clearAllocateRequestLists();
}
+ @After
+ public void cleanup() {
+ this.relayer.shutdown();
+ }
+
private void assertAsksAndReleases(int expectedAsk, int expectedRelease) {
Assert.assertEquals(expectedAsk, this.mockAMS.lastAsk.size());
Assert.assertEquals(expectedRelease, this.mockAMS.lastRelease.size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989715ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java
index ebbfae2..dd48241 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java
@@ -141,17 +141,11 @@ public class TestAMRMClientRelayerMetrics {
this.homeRelayer = new AMRMClientRelayer(this.mockAMS,
ApplicationId.newInstance(0, 0), this.homeID);
- this.homeRelayer.init(conf);
- this.homeRelayer.start();
-
this.homeRelayer.registerApplicationMaster(
RegisterApplicationMasterRequest.newInstance("", 0, ""));
this.uamRelayer = new AMRMClientRelayer(this.mockAMS,
ApplicationId.newInstance(0, 0), this.uamID);
- this.uamRelayer.init(conf);
- this.uamRelayer.start();
-
this.uamRelayer.registerApplicationMaster(
RegisterApplicationMasterRequest.newInstance("", 0, ""));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989715ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
index 54e7dd3..abb1d93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
@@ -87,7 +87,7 @@ public class TestUnmanagedApplicationManager {
}
}
- @Test(timeout = 5000)
+ @Test(timeout = 10000)
public void testBasicUsage()
throws YarnException, IOException, InterruptedException {
@@ -104,6 +104,11 @@ public class TestUnmanagedApplicationManager {
finishApplicationMaster(
FinishApplicationMasterRequest.newInstance(null, null, null),
attemptId);
+
+ while (uam.isHeartbeatThreadAlive()) {
+ LOG.info("waiting for heartbeat thread to finish");
+ Thread.sleep(100);
+ }
}
/*
@@ -261,7 +266,7 @@ public class TestUnmanagedApplicationManager {
attemptId);
}
- @Test
+ @Test(timeout = 10000)
public void testForceKill()
throws YarnException, IOException, InterruptedException {
launchUAM(attemptId);
@@ -269,6 +274,11 @@ public class TestUnmanagedApplicationManager {
RegisterApplicationMasterRequest.newInstance(null, 0, null), attemptId);
uam.forceKillApplication();
+ while (uam.isHeartbeatThreadAlive()) {
+ LOG.info("waiting for heartbeat thread to finish");
+ Thread.sleep(100);
+ }
+
try {
uam.forceKillApplication();
Assert.fail("Should fail because application is already killed");
@@ -276,6 +286,19 @@ public class TestUnmanagedApplicationManager {
}
}
+ @Test(timeout = 10000)
+ public void testShutDownConnections()
+ throws YarnException, IOException, InterruptedException {
+ launchUAM(attemptId);
+ registerApplicationMaster(
+ RegisterApplicationMasterRequest.newInstance(null, 0, null), attemptId);
+ uam.shutDownConnections();
+ while (uam.isHeartbeatThreadAlive()) {
+ LOG.info("waiting for heartbeat thread to finish");
+ Thread.sleep(100);
+ }
+ }
+
protected UserGroupInformation getUGIWithToken(
ApplicationAttemptId appAttemptId) {
UserGroupInformation ugi =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989715ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index c478871..ae9f78d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -716,12 +716,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
uamPool.finishApplicationMaster(subClusterId, finishRequest);
if (uamResponse.getIsUnregistered()) {
- AMRMClientRelayer relayer =
- secondaryRelayers.remove(subClusterId);
- if(relayer != null) {
- relayer.shutdown();
- }
-
+ secondaryRelayers.remove(subClusterId);
if (getNMStateStore() != null) {
getNMStateStore().removeAMRMProxyAppContextEntry(attemptId,
NMSS_SECONDARY_SC_PREFIX + subClusterId);
@@ -801,8 +796,16 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
*/
@Override
public void shutdown() {
+ LOG.info("Shutting down FederationInterceptor for {}", this.attemptId);
+
// Do not stop uamPool service and kill UAMs here because of possible second
// app attempt
+ try {
+ this.uamPool.shutDownConnections();
+ } catch (YarnException e) {
+ LOG.error("Error shutting down all UAM clients without killing them", e);
+ }
+
if (this.threadpool != null) {
try {
this.threadpool.shutdown();
@@ -814,9 +817,6 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
// Stop the home heartbeat thread
this.homeHeartbeartHandler.shutdown();
this.homeRMRelayer.shutdown();
- for (AMRMClientRelayer relayer : this.secondaryRelayers.values()) {
- relayer.shutdown();
- }
super.shutdown();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/989715ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
index 1eefbd5..b269fa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
@@ -206,7 +206,6 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest {
finishApplicationMaster(testAppId, FinalApplicationStatus.FAILED);
Assert.assertNotNull(finshResponse);
- Assert.assertEquals(false, finshResponse.getIsUnregistered());
try {
// Try to finish an application master that is already finished.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HADOOP-15900. Update JSch versions
in LICENSE.txt.
Posted by ae...@apache.org.
HADOOP-15900. Update JSch versions in LICENSE.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d43cc5db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d43cc5db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d43cc5db
Branch: refs/heads/HDDS-4
Commit: d43cc5db0ff80958ca873df1dc2fa00054e37175
Parents: a5e21cd
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Nov 5 17:51:16 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Nov 5 17:51:26 2018 +0900
----------------------------------------------------------------------
LICENSE.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d43cc5db/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 1a97528..81eb32f 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1836,7 +1836,7 @@ any resulting litigation.
The binary distribution of this product bundles these dependencies under the
following license:
-JSch 0.1.51
+JSch 0.1.54
ParaNamer Core 2.3
JLine 0.9.94
leveldbjni-all 1.8
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-8954. Reservations list field
in ReservationListInfo is not accessible. Contributed by Oleksandr
Shevchenko.
Posted by ae...@apache.org.
YARN-8954. Reservations list field in ReservationListInfo is not accessible. Contributed by Oleksandr Shevchenko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/babc946d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/babc946d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/babc946d
Branch: refs/heads/HDDS-4
Commit: babc946d4017e9c385d19a8e6f7f1ecd5080d619
Parents: 44e37b4
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 2 11:10:08 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Nov 2 11:10:08 2018 -0700
----------------------------------------------------------------------
.../server/resourcemanager/webapp/dao/ReservationListInfo.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/babc946d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationListInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationListInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationListInfo.java
index 25df67a..e0e65e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationListInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationListInfo.java
@@ -50,4 +50,8 @@ public class ReservationListInfo {
includeResourceAllocations));
}
}
+
+ public List<ReservationInfo> getReservations() {
+ return reservations;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-788. Change title page of
bucket browser in S3gateway. Contributed by Bharat Viswanadham.
Posted by ae...@apache.org.
HDDS-788. Change title page of bucket browser in S3gateway. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d9c18cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d9c18cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d9c18cf
Branch: refs/heads/HDDS-4
Commit: 6d9c18cfa9a458423b832a59166a15d098281ccd
Parents: d16d5f7
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Fri Nov 2 08:00:18 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Fri Nov 2 08:00:18 2018 -0700
----------------------------------------------------------------------
.../s3gateway/src/main/resources/browser.html | 4 ++--
.../resources/webapps/s3gateway/WEB-INF/web.xml | 4 ++++
.../main/resources/webapps/static/images/ozone.ico | Bin 0 -> 1150 bytes
3 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d9c18cf/hadoop-ozone/s3gateway/src/main/resources/browser.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/resources/browser.html b/hadoop-ozone/s3gateway/src/main/resources/browser.html
index dc05a00..a1f2338 100644
--- a/hadoop-ozone/s3gateway/src/main/resources/browser.html
+++ b/hadoop-ozone/s3gateway/src/main/resources/browser.html
@@ -19,10 +19,10 @@ permissions and limitations under the License.
<html lang="en">
<head>
- <title>AWS S3 Explorer</title>
+ <title>Ozone S3 Explorer</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link rel="shortcut icon" href="https://aws.amazon.com/favicon.ico">
+ <link rel="shortcut icon" href="static/images/ozone.ico">
<link rel="stylesheet"
href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
<link rel="stylesheet"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d9c18cf/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml b/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml
index a3552f0..36aad1c 100644
--- a/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml
+++ b/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml
@@ -21,6 +21,10 @@
<param-name>javax.ws.rs.Application</param-name>
<param-value>org.apache.hadoop.ozone.s3.GatewayApplication</param-value>
</init-param>
+ <init-param>
+ <param-name>jersey.config.servlet.filter.staticContentRegex</param-name>
+ <param-value>/static/images/*.ico</param-value>
+ </init-param>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d9c18cf/hadoop-ozone/s3gateway/src/main/resources/webapps/static/images/ozone.ico
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/resources/webapps/static/images/ozone.ico b/hadoop-ozone/s3gateway/src/main/resources/webapps/static/images/ozone.ico
new file mode 100755
index 0000000..72886ea
Binary files /dev/null and b/hadoop-ozone/s3gateway/src/main/resources/webapps/static/images/ozone.ico differ
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDDS-697. update and validate the
BCSID for PutSmallFile/GetSmallFile command. Contributed by Shashikant
Banerjee.
Posted by ae...@apache.org.
HDDS-697. update and validate the BCSID for PutSmallFile/GetSmallFile command. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b13c5674
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b13c5674
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b13c5674
Branch: refs/heads/HDDS-4
Commit: b13c56742a6fc0f6cb1ddd63e1afd51eb216e052
Parents: c5eb237
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Thu Nov 1 10:21:25 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Thu Nov 1 10:21:39 2018 +0530
----------------------------------------------------------------------
.../scm/storage/ContainerProtocolCalls.java | 8 ++-
.../main/proto/DatanodeContainerProtocol.proto | 2 +-
.../server/ratis/ContainerStateMachine.java | 24 ++++++---
.../container/keyvalue/KeyValueHandler.java | 2 -
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../keyvalue/helpers/SmallFileUtils.java | 7 +++
.../rpc/TestContainerStateMachineFailures.java | 21 +-------
.../ozone/scm/TestContainerSmallFile.java | 51 ++++++++++++++++++++
8 files changed, 85 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b13c5674/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
index 150b1d6..c1d90a5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
@@ -59,6 +59,8 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.WriteChunkRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
+ PutSmallFileResponseProto;
import org.apache.hadoop.hdds.client.BlockID;
import java.io.IOException;
@@ -231,10 +233,11 @@ public final class ContainerProtocolCalls {
* @param blockID - ID of the block
* @param data - Data to be written into the container.
* @param traceID - Trace ID for logging purpose.
+ * @return container protocol writeSmallFile response
* @throws IOException
*/
- public static void writeSmallFile(XceiverClientSpi client,
- BlockID blockID, byte[] data, String traceID)
+ public static PutSmallFileResponseProto writeSmallFile(
+ XceiverClientSpi client, BlockID blockID, byte[] data, String traceID)
throws IOException {
BlockData containerBlockData =
@@ -268,6 +271,7 @@ public final class ContainerProtocolCalls {
.build();
ContainerCommandResponseProto response = client.sendCommand(request);
validateContainerResponse(response);
+ return response.getPutSmallFile();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b13c5674/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 1700e23..df26f24 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -413,7 +413,7 @@ message PutSmallFileRequestProto {
message PutSmallFileResponseProto {
-
+ required GetCommittedBlockLengthResponseProto committedBlockLength = 1;
}
message GetSmallFileRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b13c5674/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index ac0833b..d5762bc 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -481,10 +481,11 @@ public class ContainerStateMachine extends BaseStateMachine {
getRequestProto(trx.getStateMachineLogEntry().getLogData());
Type cmdType = requestProto.getCmdType();
CompletableFuture<Message> future;
- if (cmdType == Type.PutBlock) {
+ if (cmdType == Type.PutBlock || cmdType == Type.PutSmallFile) {
BlockData blockData;
- ContainerProtos.BlockData blockDataProto =
- requestProto.getPutBlock().getBlockData();
+ ContainerProtos.BlockData blockDataProto = cmdType == Type.PutBlock ?
+ requestProto.getPutBlock().getBlockData() :
+ requestProto.getPutSmallFile().getBlock().getBlockData();
// set the blockCommitSequenceId
try {
@@ -499,9 +500,20 @@ public class ContainerStateMachine extends BaseStateMachine {
ContainerProtos.PutBlockRequestProto
.newBuilder(requestProto.getPutBlock())
.setBlockData(blockData.getProtoBufMessage()).build();
- ContainerCommandRequestProto containerCommandRequestProto =
- ContainerCommandRequestProto.newBuilder(requestProto)
- .setPutBlock(putBlockRequestProto).build();
+ ContainerCommandRequestProto containerCommandRequestProto;
+ if (cmdType == Type.PutSmallFile) {
+ ContainerProtos.PutSmallFileRequestProto smallFileRequestProto =
+ ContainerProtos.PutSmallFileRequestProto
+ .newBuilder(requestProto.getPutSmallFile())
+ .setBlock(putBlockRequestProto).build();
+ containerCommandRequestProto =
+ ContainerCommandRequestProto.newBuilder(requestProto)
+ .setPutSmallFile(smallFileRequestProto).build();
+ } else {
+ containerCommandRequestProto =
+ ContainerCommandRequestProto.newBuilder(requestProto)
+ .setPutBlock(putBlockRequestProto).build();
+ }
future = CompletableFuture
.supplyAsync(() -> runCommand(containerCommandRequestProto),
getCommandExecutor(requestProto));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b13c5674/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 7aaa5e6..4cb23ed 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -756,8 +756,6 @@ public class KeyValueHandler extends Handler {
try {
BlockID blockID = BlockID.getFromProtobuf(getSmallFileReq.getBlock()
.getBlockID());
- // TODO: add bcsId as a part of getSmallFile transaction
- // by default its 0
BlockData responseData = blockManager.getBlock(kvContainer, blockID);
ContainerProtos.ChunkInfo chunkInfo = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b13c5674/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
index e085fb0..667e66d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
@@ -188,7 +188,7 @@ public final class BlockUtils {
return builder.build();
}
- private static GetCommittedBlockLengthResponseProto.Builder
+ public static GetCommittedBlockLengthResponseProto.Builder
getCommittedBlockLengthResponseBuilder(long blockLength,
ContainerProtos.DatanodeBlockID blockID) {
ContainerProtos.GetCommittedBlockLengthResponseProto.Builder
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b13c5674/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
index dee0c11..e91c8a6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
@@ -47,6 +47,13 @@ public final class SmallFileUtils {
ContainerCommandRequestProto msg) {
ContainerProtos.PutSmallFileResponseProto.Builder getResponse =
ContainerProtos.PutSmallFileResponseProto.newBuilder();
+ ContainerProtos.BlockData blockData =
+ msg.getPutSmallFile().getBlock().getBlockData();
+ ContainerProtos.GetCommittedBlockLengthResponseProto.Builder
+ committedBlockLengthResponseBuilder = BlockUtils
+ .getCommittedBlockLengthResponseBuilder(blockData.getSize(),
+ blockData.getBlockID());
+ getResponse.setCommittedBlockLength(committedBlockLengthResponseBuilder);
ContainerCommandResponseProto.Builder builder =
ContainerUtils.getSuccessResponseBuilder(msg);
builder.setCmdType(ContainerProtos.Type.PutSmallFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b13c5674/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
index b3f0be7..0640649 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
@@ -23,16 +23,9 @@ import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.
- StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto.
- StorageContainerDatanodeProtocolProtos.ContainerAction.Action;
-import org.apache.hadoop.hdds.protocol.proto.
- StorageContainerDatanodeProtocolProtos.ContainerAction.Reason;
import org.apache.hadoop.hdds.scm.container.
common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
@@ -88,7 +81,6 @@ public class TestContainerStateMachineFailures {
File baseDir = new File(path);
baseDir.mkdirs();
- chunkSize = (int) OzoneConsts.MB;
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
TimeUnit.MILLISECONDS);
@@ -140,7 +132,6 @@ public class TestContainerStateMachineFailures {
Assert.assertEquals(1, locationInfoList.size());
OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
- long containerID = omKeyLocationInfo.getContainerID();
// delete the container dir
FileUtil.fullyDelete(new File(
cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
@@ -171,15 +162,5 @@ public class TestContainerStateMachineFailures {
Assert.assertTrue(((StorageContainerException) ioe.getCause()).getResult()
== ContainerProtos.Result.CONTAINER_UNHEALTHY);
}
- StorageContainerDatanodeProtocolProtos.ContainerAction action =
- StorageContainerDatanodeProtocolProtos.ContainerAction.newBuilder()
- .setContainerID(containerID).setAction(Action.CLOSE)
- .setReason(Reason.CONTAINER_UNHEALTHY)
- .build();
-
- // Make sure the container close action is initiated to SCM.
- Assert.assertTrue(
- cluster.getHddsDatanodes().get(0).getDatanodeStateMachine().getContext()
- .getAllPendingContainerActions().contains(action));
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b13c5674/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
index 90dc2c4..ecf0d84 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
@@ -152,7 +152,58 @@ public class TestContainerSmallFile {
xceiverClientManager.releaseClient(client);
}
+ @Test
+ public void testReadWriteWithBCSId() throws Exception {
+ String traceID = UUID.randomUUID().toString();
+ ContainerWithPipeline container =
+ storageContainerLocationClient.allocateContainer(
+ HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.ONE, containerOwner);
+ XceiverClientSpi client = xceiverClientManager
+ .acquireClient(container.getPipeline());
+ ContainerProtocolCalls.createContainer(client,
+ container.getContainerInfo().getContainerID(), traceID);
+ BlockID blockID1 = ContainerTestHelper.getTestBlockID(
+ container.getContainerInfo().getContainerID());
+ ContainerProtos.PutSmallFileResponseProto responseProto =
+ ContainerProtocolCalls
+ .writeSmallFile(client, blockID1, "data123".getBytes(), traceID);
+ long bcsId = responseProto.getCommittedBlockLength().getBlockID()
+ .getBlockCommitSequenceId();
+ try {
+ blockID1.setBlockCommitSequenceId(bcsId + 1);
+ //read a file with higher bcsId than the container bcsId
+ ContainerProtocolCalls
+ .readSmallFile(client, blockID1, traceID);
+ Assert.fail("Expected exception not thrown");
+ } catch (StorageContainerException sce) {
+ Assert
+ .assertTrue(sce.getResult() == ContainerProtos.Result.UNKNOWN_BCSID);
+ }
+
+ // write a new block again to bump up the container bcsId
+ BlockID blockID2 = ContainerTestHelper
+ .getTestBlockID(container.getContainerInfo().getContainerID());
+ ContainerProtocolCalls
+ .writeSmallFile(client, blockID2, "data123".getBytes(), traceID);
+
+ try {
+ blockID1.setBlockCommitSequenceId(bcsId + 1);
+ //read a file with higher bcsId than the committed bcsId for the block
+ ContainerProtocolCalls.readSmallFile(client, blockID1, traceID);
+ Assert.fail("Expected exception not thrown");
+ } catch (StorageContainerException sce) {
+ Assert
+ .assertTrue(sce.getResult() == ContainerProtos.Result.BCSID_MISMATCH);
+ }
+ blockID1.setBlockCommitSequenceId(bcsId);
+ ContainerProtos.GetSmallFileResponseProto response =
+ ContainerProtocolCalls.readSmallFile(client, blockID1, traceID);
+ String readData = response.getData().getData().toStringUtf8();
+ Assert.assertEquals("data123", readData);
+ xceiverClientManager.releaseClient(client);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HDDS-794. Add configs to set
StateMachineData write timeout in ContainerStateMachine. Contributed by
Shashikant Banerjee.
Posted by ae...@apache.org.
HDDS-794. Add configs to set StateMachineData write timeout in ContainerStateMachine. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ddefdd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ddefdd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ddefdd5
Branch: refs/heads/HDDS-4
Commit: 5ddefdd50751ed316f2eb9046f294bbdcdfb2428
Parents: 942693b
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Nov 5 10:10:10 2018 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Nov 5 10:41:28 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 ++++++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 9 +++++++++
.../common/src/main/resources/ozone-default.xml | 7 +++++++
.../server/ratis/ContainerStateMachine.java | 18 ++++++++++++++++--
.../server/ratis/XceiverServerRatis.java | 14 ++++++++++++++
.../container/keyvalue/helpers/ChunkUtils.java | 2 ++
.../container/keyvalue/impl/ChunkManagerImpl.java | 3 ++-
7 files changed, 56 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 56692af..38eec61 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -79,6 +79,12 @@ public final class ScmConfigKeys {
"dfs.container.ratis.segment.preallocated.size";
public static final int
DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
+ "dfs.container.ratis.statemachinedata.sync.timeout";
+ public static final TimeDuration
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
+ TimeDuration.valueOf(10, TimeUnit.SECONDS);
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
"dfs.ratis.client.request.timeout.duration";
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 3b4f017..54b1cf8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -229,6 +229,15 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
+
+ // config settings to enable stateMachineData write timeout
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT;
+ public static final TimeDuration
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
+
public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
= ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index eb68662..5ff60eb 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -53,6 +53,13 @@
</description>
</property>
<property>
+ <name>dfs.container.ratis.statemachinedata.sync.timeout</name>
+ <value>10s</value>
+ <tag>OZONE, DEBUG, CONTAINER, RATIS</tag>
+ <description>Timeout for StateMachine data writes by Ratis.
+ </description>
+ </property>
+ <property>
<name>dfs.container.ratis.datanode.storage.dir</name>
<value/>
<tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index d5762bc..2a4a227 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -307,9 +307,18 @@ public class ContainerStateMachine extends BaseStateMachine {
() -> runCommand(requestProto), chunkExecutor);
}
writeChunkFutureMap.put(entryIndex, writeChunkFuture);
+ LOG.debug("writeChunk writeStateMachineData : blockId " + write.getBlockID()
+ + " logIndex " + entryIndex + " chunkName " + write.getChunkData()
+ .getChunkName());
// Remove the future once it finishes execution from the
// writeChunkFutureMap.
- writeChunkFuture.thenApply(r -> writeChunkFutureMap.remove(entryIndex));
+ writeChunkFuture.thenApply(r -> {
+ writeChunkFutureMap.remove(entryIndex);
+ LOG.debug("writeChunk writeStateMachineData completed: blockId " + write
+ .getBlockID() + " logIndex " + entryIndex + " chunkName " + write
+ .getChunkData().getChunkName());
+ return r;
+ });
return writeChunkFuture;
}
@@ -531,7 +540,12 @@ public class ContainerStateMachine extends BaseStateMachine {
if (cmdType == Type.CreateContainer) {
long containerID = requestProto.getContainerID();
future.thenApply(
- r -> createContainerFutureMap.remove(containerID).complete(null));
+ r -> {
+ createContainerFutureMap.remove(containerID).complete(null);
+ LOG.info("create Container Transaction completed for container " +
+ containerID + " log index " + index);
+ return r;
+ });
}
future.thenAccept(m -> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 599f821..a679e5e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -186,6 +186,20 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftClientConfigKeys.Rpc
.setRequestTimeout(properties, clientRequestTimeout);
+ // set the configs enable and set the stateMachineData sync timeout
+ RaftServerConfigKeys.Log.StateMachineData.setSync(properties, true);
+ timeUnit = OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT.getUnit();
+ duration = conf.getTimeDuration(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT,
+ OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT
+ .getDuration(), timeUnit);
+ final TimeDuration dataSyncTimeout =
+ TimeDuration.valueOf(duration, timeUnit);
+ RaftServerConfigKeys.Log.StateMachineData
+ .setSyncTimeout(properties, dataSyncTimeout);
+
// Set the server Request timeout
timeUnit = OzoneConfigKeys.DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_DEFAULT
.getUnit();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 718f5de..8f9d589 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -138,6 +138,8 @@ public final class ChunkUtils {
}
}
}
+ log.debug("Write Chunk completed for chunkFile: {}, size {}", chunkFile,
+ data.length);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ddefdd5/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index c630e19..cdd19df 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -82,7 +82,8 @@ public class ChunkManagerImpl implements ChunkManager {
chunkFile, info);
File tmpChunkFile = getTmpChunkFile(chunkFile, info);
- LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file",
+ LOG.debug(
+ "writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file:{}",
info.getChunkName(), stage, chunkFile, tmpChunkFile);
switch (stage) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-8854. Upgrade jquery
datatable version references to v1.10.19. Contributed by Akhil PB.
Posted by ae...@apache.org.
YARN-8854. Upgrade jquery datatable version references to v1.10.19. Contributed by Akhil PB.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d36012b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d36012b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d36012b6
Branch: refs/heads/HDDS-4
Commit: d36012b69f01c9ddfd2e95545d1f5e1fbc1c3236
Parents: 62d98ca
Author: Sunil G <su...@apache.org>
Authored: Tue Oct 30 22:56:13 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Oct 30 22:56:46 2018 +0530
----------------------------------------------------------------------
LICENSE.txt | 2 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 10 +-
.../hadoop/yarn/webapp/view/JQueryUI.java | 6 +-
.../static/dt-1.10.18/css/custom_datatable.css | 68 +++
.../webapps/static/dt-1.10.18/css/demo_page.css | 108 ++++
.../static/dt-1.10.18/css/demo_table.css | 544 +++++++++++++++++++
.../static/dt-1.10.18/css/jquery.dataTables.css | 466 ++++++++++++++++
.../webapps/static/dt-1.10.18/css/jui-dt.css | 352 ++++++++++++
.../static/dt-1.10.18/images/Sorting icons.psd | Bin 0 -> 27490 bytes
.../static/dt-1.10.18/images/back_disabled.jpg | Bin 0 -> 612 bytes
.../static/dt-1.10.18/images/back_enabled.jpg | Bin 0 -> 807 bytes
.../static/dt-1.10.18/images/favicon.ico | Bin 0 -> 894 bytes
.../dt-1.10.18/images/forward_disabled.jpg | Bin 0 -> 635 bytes
.../dt-1.10.18/images/forward_enabled.jpg | Bin 0 -> 852 bytes
.../static/dt-1.10.18/images/sort_asc.png | Bin 0 -> 263 bytes
.../dt-1.10.18/images/sort_asc_disabled.png | Bin 0 -> 252 bytes
.../static/dt-1.10.18/images/sort_both.png | Bin 0 -> 282 bytes
.../static/dt-1.10.18/images/sort_desc.png | Bin 0 -> 260 bytes
.../dt-1.10.18/images/sort_desc_disabled.png | Bin 0 -> 251 bytes
.../dt-1.10.18/js/jquery.dataTables.min.js | 184 +++++++
.../webapps/static/dt-1.10.7/css/demo_page.css | 110 ----
.../webapps/static/dt-1.10.7/css/demo_table.css | 538 ------------------
.../webapps/static/dt-1.10.7/css/jui-dt.css | 322 -----------
.../static/dt-1.10.7/images/Sorting icons.psd | Bin 27490 -> 0 bytes
.../static/dt-1.10.7/images/back_disabled.jpg | Bin 612 -> 0 bytes
.../static/dt-1.10.7/images/back_enabled.jpg | Bin 807 -> 0 bytes
.../webapps/static/dt-1.10.7/images/favicon.ico | Bin 894 -> 0 bytes
.../dt-1.10.7/images/forward_disabled.jpg | Bin 635 -> 0 bytes
.../static/dt-1.10.7/images/forward_enabled.jpg | Bin 852 -> 0 bytes
.../static/dt-1.10.7/images/sort_asc.png | Bin 263 -> 0 bytes
.../dt-1.10.7/images/sort_asc_disabled.png | Bin 252 -> 0 bytes
.../static/dt-1.10.7/images/sort_both.png | Bin 282 -> 0 bytes
.../static/dt-1.10.7/images/sort_desc.png | Bin 260 -> 0 bytes
.../dt-1.10.7/images/sort_desc_disabled.png | Bin 251 -> 0 bytes
.../dt-1.10.7/js/jquery.dataTables.min.js | 160 ------
35 files changed, 1733 insertions(+), 1137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 94c9065..1a97528 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -553,7 +553,7 @@ For:
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.7/
+hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/
--------------------------------------------------------------------------------
Copyright (C) 2008-2016, SpryMedia Ltd.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 133003a..641a5f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -237,10 +237,12 @@
<exclude>src/main/resources/webapps/test/.keep</exclude>
<exclude>src/main/resources/webapps/proxy/.keep</exclude>
<exclude>src/main/resources/webapps/node/.keep</exclude>
- <exclude>src/main/resources/webapps/static/dt-1.10.7/css/jui-dt.css</exclude>
- <exclude>src/main/resources/webapps/static/dt-1.10.7/css/demo_table.css</exclude>
- <exclude>src/main/resources/webapps/static/dt-1.10.7/images/Sorting icons.psd</exclude>
- <exclude>src/main/resources/webapps/static/dt-1.10.7/js/jquery.dataTables.min.js</exclude>
+ <exclude>src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css</exclude>
+ <exclude>src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css</exclude>
+ <exclude>src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css</exclude>
+ <exclude>src/main/resources/webapps/static/dt-1.10.18/css/demo_table.css</exclude>
+ <exclude>src/main/resources/webapps/static/dt-1.10.18/images/Sorting icons.psd</exclude>
+ <exclude>src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js</exclude>
<exclude>src/main/resources/webapps/static/jt/jquery.jstree.js</exclude>
<exclude>src/main/resources/webapps/static/jquery/jquery-ui-1.12.1.custom.min.js</exclude>
<exclude>src/main/resources/webapps/static/jquery/jquery-3.3.1.min.js</exclude>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
index eef33eb..0754c36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
@@ -66,10 +66,12 @@ public class JQueryUI extends HtmlBlock {
@Override
protected void render(Block html) {
html.link(root_url("static/jquery/themes-1.9.1/base/jquery-ui.css"))
- .link(root_url("static/dt-1.10.7/css/jui-dt.css"))
+ .link(root_url("static/dt-1.10.18/css/jquery.dataTables.css"))
+ .link(root_url("static/dt-1.10.18/css/jui-dt.css"))
+ .link(root_url("static/dt-1.10.18/css/custom_datatable.css"))
.script(root_url("static/jquery/jquery-3.3.1.min.js"))
.script(root_url("static/jquery/jquery-ui-1.12.1.custom.min.js"))
- .script(root_url("static/dt-1.10.7/js/jquery.dataTables.min.js"))
+ .script(root_url("static/dt-1.10.18/js/jquery.dataTables.min.js"))
.script(root_url("static/yarn.dt.plugins.js"))
.script(root_url("static/dt-sorting/natural.js"))
.style("#jsnotice { padding: 0.2em; text-align: center; }",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css
new file mode 100644
index 0000000..1339d67
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Fixing broken datatable styles after
+ * upgrading jquery.dataTables.min.js to v1.10.18
+ */
+
+.dataTables_wrapper {
+ border: 1px solid #aaaaaa;
+}
+
+.dataTables_wrapper .dataTables_length,
+.dataTables_wrapper .dataTables_filter,
+.dataTables_wrapper .dataTables_info,
+.dataTables_wrapper .dataTables_paginate {
+ padding: 5px;
+}
+
+.dataTables_wrapper .dataTables_length,
+.dataTables_wrapper .dataTables_filter {
+ width: 50%;
+ height: 27px;
+ box-sizing: border-box;
+ background-color: #cccccc;
+}
+
+.dataTables_wrapper .dataTables_info {
+ color: #222222;
+ padding: 13px 5px;
+}
+
+table.dataTable thead th,
+table.dataTable thead td {
+ color: #555555;
+ border-top: 1px solid #aaaaaa;
+ border-bottom: 1px solid #aaaaaa;
+ border-right: 1px solid #aaaaaa;
+ background-color: #e6e6e6;
+}
+
+table.dataTable thead th:last-of-type,
+table.dataTable thead td:last-of-type {
+ border-right: none;
+}
+
+table.dataTable.no-footer {
+ border-bottom: 1px solid #aaaaaa;
+}
+
+table.dataTable tbody tr.even {
+ background: rgba(0, 0, 255, 0.05);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_page.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_page.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_page.css
new file mode 100644
index 0000000..38e4aee
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_page.css
@@ -0,0 +1,108 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * General page setup
+ */
+#dt_example {
+ font: 80%/1.45em "Lucida Grande", Verdana, Arial, Helvetica, sans-serif;
+ margin: 0;
+ padding: 0;
+ color: #333;
+ background-color: #fff;
+}
+
+#dt_example #container {
+ width: 800px;
+ margin: 30px auto;
+ padding: 0;
+}
+
+#dt_example #footer {
+ margin: 50px auto 0 auto;
+ padding: 0;
+}
+
+#dt_example #demo {
+ margin: 30px auto 0 auto;
+}
+
+#dt_example .demo_jui {
+ margin: 30px auto 0 auto;
+}
+
+#dt_example .big {
+ font-size: 1.3em;
+ font-weight: bold;
+ line-height: 1.6em;
+ color: #4e6ca3;
+}
+
+#dt_example .spacer {
+ height: 20px;
+ clear: both;
+}
+
+#dt_example .clear {
+ clear: both;
+}
+
+#dt_example pre {
+ padding: 15px;
+ background-color: #f5f5f5;
+ border: 1px solid #cccccc;
+}
+
+#dt_example h1 {
+ margin-top: 2em;
+ font-size: 1.3em;
+ font-weight: normal;
+ line-height: 1.6em;
+ color: #4e6ca3;
+ border-bottom: 1px solid #b0bed9;
+ clear: both;
+}
+
+#dt_example h2 {
+ font-size: 1.2em;
+ font-weight: normal;
+ line-height: 1.6em;
+ color: #4e6ca3;
+ clear: both;
+}
+
+#dt_example a {
+ color: #0063dc;
+ text-decoration: none;
+}
+
+#dt_example a:hover {
+ text-decoration: underline;
+}
+
+#dt_example ul {
+ color: #4e6ca3;
+}
+
+.css_right {
+ float: right;
+}
+
+.css_left {
+ float: left;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_table.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_table.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_table.css
new file mode 100644
index 0000000..cb4f021
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_table.css
@@ -0,0 +1,544 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+ * File: demo_table.css
+ * CVS: $Id$
+ * Description: CSS descriptions for DataTables demo pages
+ * Author: Allan Jardine
+ * Created: Tue May 12 06:47:22 BST 2009
+ * Modified: $Date$ by $Author$
+ * Language: CSS
+ * Project: DataTables
+ *
+ * Copyright 2009 Allan Jardine. All Rights Reserved.
+ *
+ * ***************************************************************************
+ * DESCRIPTION
+ *
+ * The styles given here are suitable for the demos that are used with the standard DataTables
+ * distribution (see www.datatables.net). You will most likely wish to modify these styles to
+ * meet the layout requirements of your site.
+ *
+ * Common issues:
+ * 'full_numbers' pagination - I use an extra selector on the body tag to ensure that there is
+ * no conflict between the two pagination types. If you want to use full_numbers pagination
+ * ensure that you either have "example_alt_pagination" as a body class name, or better yet,
+ * modify that selector.
+ * Note that the path used for Images is relative. All images are by default located in
+ * ../images/ - relative to this CSS file.
+ */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables features
+ */
+
+.dataTables_wrapper {
+ position: relative;
+ min-height: 302px;
+ clear: both;
+ _height: 302px;
+ zoom: 1; /* Feeling sorry for IE */
+}
+
+.dataTables_processing {
+ position: absolute;
+ top: 50%;
+ left: 50%;
+ width: 250px;
+ height: 30px;
+ margin-left: -125px;
+ margin-top: -15px;
+ padding: 14px 0 2px 0;
+ border: 1px solid #ddd;
+ text-align: center;
+ color: #999;
+ font-size: 14px;
+ background-color: white;
+}
+
+.dataTables_length {
+ width: 40%;
+ float: left;
+}
+
+.dataTables_filter {
+ width: 50%;
+ float: right;
+ text-align: right;
+}
+
+.dataTables_info {
+ width: 60%;
+ float: left;
+}
+
+.dataTables_paginate {
+ width: 44px;
+ *width: 50px;
+ float: right;
+ text-align: right;
+}
+
+/* Pagination nested */
+.paginate_disabled_previous,
+.paginate_enabled_previous,
+.paginate_disabled_next,
+.paginate_enabled_next {
+ height: 19px;
+ width: 19px;
+ margin-left: 3px;
+ float: left;
+}
+
+.paginate_disabled_previous {
+ background-image: url("../images/back_disabled.jpg");
+}
+
+.paginate_enabled_previous {
+ background-image: url("../images/back_enabled.jpg");
+}
+
+.paginate_disabled_next {
+ background-image: url("../images/forward_disabled.jpg");
+}
+
+.paginate_enabled_next {
+ background-image: url("../images/forward_enabled.jpg");
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables display
+ */
+table.display {
+ margin: 0 auto;
+ clear: both;
+ width: 100%;
+
+ /* Note Firefox 3.5 and before have a bug with border-collapse
+ * ( https://bugzilla.mozilla.org/show%5Fbug.cgi?id=155955 )
+ * border-spacing: 0; is one possible option. Conditional-css.com is
+ * useful for this kind of thing
+ *
+ * Further note IE 6/7 has problems when calculating widths with border width.
+ * It subtracts one px relative to the other browsers from the first column, and
+ * adds one to the end...
+ *
+ * If you want that effect I'd suggest setting a border-top/left on th/td's and
+ * then filling in the gaps with other borders.
+ */
+}
+
+table.display thead th {
+ padding: 3px 18px 3px 10px;
+ border-bottom: 1px solid black;
+ font-weight: bold;
+ cursor: pointer;
+ *cursor: hand;
+}
+
+table.display tfoot th {
+ padding: 3px 18px 3px 10px;
+ border-top: 1px solid black;
+ font-weight: bold;
+}
+
+table.display tr.heading2 td {
+ border-bottom: 1px solid #aaa;
+}
+
+table.display td {
+ padding: 3px 10px;
+}
+
+table.display td.center {
+ text-align: center;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables sorting
+ */
+
+.sorting_asc {
+ background: url("../images/sort_asc.png") no-repeat center right;
+}
+
+.sorting_desc {
+ background: url("../images/sort_desc.png") no-repeat center right;
+}
+
+.sorting {
+ background: url("../images/sort_both.png") no-repeat center right;
+}
+
+.sorting_asc_disabled {
+ background: url("../images/sort_asc_disabled.png") no-repeat center right;
+}
+
+.sorting_desc_disabled {
+ background: url("../images/sort_desc_disabled.png") no-repeat center right;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables row classes
+ */
+table.display tr.odd.gradeA {
+ background-color: #ddffdd;
+}
+
+table.display tr.even.gradeA {
+ background-color: #eeffee;
+}
+
+table.display tr.odd.gradeC {
+ background-color: #ddddff;
+}
+
+table.display tr.even.gradeC {
+ background-color: #eeeeff;
+}
+
+table.display tr.odd.gradeX {
+ background-color: #ffdddd;
+}
+
+table.display tr.even.gradeX {
+ background-color: #ffeeee;
+}
+
+table.display tr.odd.gradeU {
+ background-color: #ddd;
+}
+
+table.display tr.even.gradeU {
+ background-color: #eee;
+}
+
+tr.odd {
+ background-color: #e2e4ff;
+}
+
+tr.even {
+ background-color: white;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Misc
+ */
+.dataTables_scroll {
+ clear: both;
+}
+
+.dataTables_scrollBody {
+ *margin-top: -1px;
+}
+
+.top,
+.bottom {
+ padding: 15px;
+ background-color: #f5f5f5;
+ border: 1px solid #cccccc;
+}
+
+.top .dataTables_info {
+ float: none;
+}
+
+.clear {
+ clear: both;
+}
+
+.dataTables_empty {
+ text-align: center;
+}
+
+tfoot input {
+ margin: 0.5em 0;
+ width: 100%;
+ color: #444;
+}
+
+tfoot input.search_init {
+ color: #999;
+}
+
+td.group {
+ background-color: #d1cfd0;
+ border-bottom: 2px solid #a19b9e;
+ border-top: 2px solid #a19b9e;
+}
+
+td.details {
+ background-color: #d1cfd0;
+ border: 2px solid #a19b9e;
+}
+
+.example_alt_pagination div.dataTables_info {
+ width: 40%;
+}
+
+.paging_full_numbers {
+ width: 400px;
+ height: 22px;
+ line-height: 22px;
+}
+
+.paging_full_numbers span.paginate_button,
+.paging_full_numbers span.paginate_active {
+ border: 1px solid #aaa;
+ -webkit-border-radius: 5px;
+ -moz-border-radius: 5px;
+ padding: 2px 5px;
+ margin: 0 3px;
+ cursor: pointer;
+ *cursor: hand;
+}
+
+.paging_full_numbers span.paginate_button {
+ background-color: #ddd;
+}
+
+.paging_full_numbers span.paginate_button:hover {
+ background-color: #ccc;
+}
+
+.paging_full_numbers span.paginate_active {
+ background-color: #99b3ff;
+}
+
+table.display tr.even.row_selected td {
+ background-color: #b0bed9;
+}
+
+table.display tr.odd.row_selected td {
+ background-color: #9fafd1;
+}
+
+/*
+ * Sorting classes for columns
+ */
+/* For the standard odd/even */
+tr.odd td.sorting_1 {
+ background-color: #d3d6ff;
+}
+
+tr.odd td.sorting_2 {
+ background-color: #dadcff;
+}
+
+tr.odd td.sorting_3 {
+ background-color: #e0e2ff;
+}
+
+tr.even td.sorting_1 {
+ background-color: #eaebff;
+}
+
+tr.even td.sorting_2 {
+ background-color: #f2f3ff;
+}
+
+tr.even td.sorting_3 {
+ background-color: #f9f9ff;
+}
+
+/* For the Conditional-CSS grading rows */
+/*
+ Colour calculations (based off the main row colours)
+ Level 1:
+ dd > c4
+ ee > d5
+ Level 2:
+ dd > d1
+ ee > e2
+ */
+tr.odd.gradeA td.sorting_1 {
+ background-color: #c4ffc4;
+}
+
+tr.odd.gradeA td.sorting_2 {
+ background-color: #d1ffd1;
+}
+
+tr.odd.gradeA td.sorting_3 {
+ background-color: #d1ffd1;
+}
+
+tr.even.gradeA td.sorting_1 {
+ background-color: #d5ffd5;
+}
+
+tr.even.gradeA td.sorting_2 {
+ background-color: #e2ffe2;
+}
+
+tr.even.gradeA td.sorting_3 {
+ background-color: #e2ffe2;
+}
+
+tr.odd.gradeC td.sorting_1 {
+ background-color: #c4c4ff;
+}
+
+tr.odd.gradeC td.sorting_2 {
+ background-color: #d1d1ff;
+}
+
+tr.odd.gradeC td.sorting_3 {
+ background-color: #d1d1ff;
+}
+
+tr.even.gradeC td.sorting_1 {
+ background-color: #d5d5ff;
+}
+
+tr.even.gradeC td.sorting_2 {
+ background-color: #e2e2ff;
+}
+
+tr.even.gradeC td.sorting_3 {
+ background-color: #e2e2ff;
+}
+
+tr.odd.gradeX td.sorting_1 {
+ background-color: #ffc4c4;
+}
+
+tr.odd.gradeX td.sorting_2 {
+ background-color: #ffd1d1;
+}
+
+tr.odd.gradeX td.sorting_3 {
+ background-color: #ffd1d1;
+}
+
+tr.even.gradeX td.sorting_1 {
+ background-color: #ffd5d5;
+}
+
+tr.even.gradeX td.sorting_2 {
+ background-color: #ffe2e2;
+}
+
+tr.even.gradeX td.sorting_3 {
+ background-color: #ffe2e2;
+}
+
+tr.odd.gradeU td.sorting_1 {
+ background-color: #c4c4c4;
+}
+
+tr.odd.gradeU td.sorting_2 {
+ background-color: #d1d1d1;
+}
+
+tr.odd.gradeU td.sorting_3 {
+ background-color: #d1d1d1;
+}
+
+tr.even.gradeU td.sorting_1 {
+ background-color: #d5d5d5;
+}
+
+tr.even.gradeU td.sorting_2 {
+ background-color: #e2e2e2;
+}
+
+tr.even.gradeU td.sorting_3 {
+ background-color: #e2e2e2;
+}
+
+/*
+ * Row highlighting example
+ */
+.ex_highlight #example tbody tr.even:hover,
+#example tbody tr.even td.highlighted {
+ background-color: #ecffb3;
+}
+
+.ex_highlight #example tbody tr.odd:hover,
+#example tbody tr.odd td.highlighted {
+ background-color: #e6ff99;
+}
+
+.ex_highlight_row #example tr.even:hover {
+ background-color: #ecffb3;
+}
+
+.ex_highlight_row #example tr.even:hover td.sorting_1 {
+ background-color: #ddff75;
+}
+
+.ex_highlight_row #example tr.even:hover td.sorting_2 {
+ background-color: #e7ff9e;
+}
+
+.ex_highlight_row #example tr.even:hover td.sorting_3 {
+ background-color: #e2ff89;
+}
+
+.ex_highlight_row #example tr.odd:hover {
+ background-color: #e6ff99;
+}
+
+.ex_highlight_row #example tr.odd:hover td.sorting_1 {
+ background-color: #d6ff5c;
+}
+
+.ex_highlight_row #example tr.odd:hover td.sorting_2 {
+ background-color: #e0ff84;
+}
+
+.ex_highlight_row #example tr.odd:hover td.sorting_3 {
+ background-color: #dbff70;
+}
+
+/*
+ * KeyTable
+ */
+table.KeyTable td {
+ border: 3px solid transparent;
+}
+
+table.KeyTable td.focus {
+ border: 3px solid #3366ff;
+}
+
+table.display tr.gradeA {
+ background-color: #eeffee;
+}
+
+table.display tr.gradeC {
+ background-color: #ddddff;
+}
+
+table.display tr.gradeX {
+ background-color: #ffdddd;
+}
+
+table.display tr.gradeU {
+ background-color: #ddd;
+}
+
+div.box {
+ height: 100px;
+ padding: 10px;
+ overflow: auto;
+ border: 1px solid #8080ff;
+ background-color: #e5e5ff;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css
new file mode 100644
index 0000000..88bf2f14
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css
@@ -0,0 +1,466 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Table styles
+ */
+table.dataTable {
+ width: 100%;
+ margin: 0 auto;
+ clear: both;
+ border-collapse: separate;
+ border-spacing: 0;
+ /*
+ * Header and footer styles
+ */
+ /*
+ * Body styles
+ */
+}
+table.dataTable thead th,
+table.dataTable tfoot th {
+ font-weight: bold;
+}
+table.dataTable thead th,
+table.dataTable thead td {
+ padding: 10px 18px;
+ border-bottom: 1px solid #111;
+}
+table.dataTable thead th:active,
+table.dataTable thead td:active {
+ outline: none;
+}
+table.dataTable tfoot th,
+table.dataTable tfoot td {
+ padding: 10px 18px 6px 18px;
+ border-top: 1px solid #111;
+}
+table.dataTable thead .sorting,
+table.dataTable thead .sorting_asc,
+table.dataTable thead .sorting_desc,
+table.dataTable thead .sorting_asc_disabled,
+table.dataTable thead .sorting_desc_disabled {
+ cursor: pointer;
+ *cursor: hand;
+ background-repeat: no-repeat;
+ background-position: center right;
+}
+table.dataTable thead .sorting {
+ background-image: url("../images/sort_both.png");
+}
+table.dataTable thead .sorting_asc {
+ background-image: url("../images/sort_asc.png");
+}
+table.dataTable thead .sorting_desc {
+ background-image: url("../images/sort_desc.png");
+}
+table.dataTable thead .sorting_asc_disabled {
+ background-image: url("../images/sort_asc_disabled.png");
+}
+table.dataTable thead .sorting_desc_disabled {
+ background-image: url("../images/sort_desc_disabled.png");
+}
+table.dataTable tbody tr {
+ background-color: #ffffff;
+}
+table.dataTable tbody tr.selected {
+ background-color: #B0BED9;
+}
+table.dataTable tbody th,
+table.dataTable tbody td {
+ padding: 8px 10px;
+}
+table.dataTable.row-border tbody th, table.dataTable.row-border tbody td, table.dataTable.display tbody th, table.dataTable.display tbody td {
+ border-top: 1px solid #ddd;
+}
+table.dataTable.row-border tbody tr:first-child th,
+table.dataTable.row-border tbody tr:first-child td, table.dataTable.display tbody tr:first-child th,
+table.dataTable.display tbody tr:first-child td {
+ border-top: none;
+}
+table.dataTable.cell-border tbody th, table.dataTable.cell-border tbody td {
+ border-top: 1px solid #ddd;
+ border-right: 1px solid #ddd;
+}
+table.dataTable.cell-border tbody tr th:first-child,
+table.dataTable.cell-border tbody tr td:first-child {
+ border-left: 1px solid #ddd;
+}
+table.dataTable.cell-border tbody tr:first-child th,
+table.dataTable.cell-border tbody tr:first-child td {
+ border-top: none;
+}
+table.dataTable.stripe tbody tr.odd, table.dataTable.display tbody tr.odd {
+ background-color: #f9f9f9;
+}
+table.dataTable.stripe tbody tr.odd.selected, table.dataTable.display tbody tr.odd.selected {
+ background-color: #acbad4;
+}
+table.dataTable.hover tbody tr:hover, table.dataTable.display tbody tr:hover {
+ background-color: #f6f6f6;
+}
+table.dataTable.hover tbody tr:hover.selected, table.dataTable.display tbody tr:hover.selected {
+ background-color: #aab7d1;
+}
+table.dataTable.order-column tbody tr > .sorting_1,
+table.dataTable.order-column tbody tr > .sorting_2,
+table.dataTable.order-column tbody tr > .sorting_3, table.dataTable.display tbody tr > .sorting_1,
+table.dataTable.display tbody tr > .sorting_2,
+table.dataTable.display tbody tr > .sorting_3 {
+ background-color: #fafafa;
+}
+table.dataTable.order-column tbody tr.selected > .sorting_1,
+table.dataTable.order-column tbody tr.selected > .sorting_2,
+table.dataTable.order-column tbody tr.selected > .sorting_3, table.dataTable.display tbody tr.selected > .sorting_1,
+table.dataTable.display tbody tr.selected > .sorting_2,
+table.dataTable.display tbody tr.selected > .sorting_3 {
+ background-color: #acbad5;
+}
+table.dataTable.display tbody tr.odd > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd > .sorting_1 {
+ background-color: #f1f1f1;
+}
+table.dataTable.display tbody tr.odd > .sorting_2, table.dataTable.order-column.stripe tbody tr.odd > .sorting_2 {
+ background-color: #f3f3f3;
+}
+table.dataTable.display tbody tr.odd > .sorting_3, table.dataTable.order-column.stripe tbody tr.odd > .sorting_3 {
+ background-color: whitesmoke;
+}
+table.dataTable.display tbody tr.odd.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_1 {
+ background-color: #a6b4cd;
+}
+table.dataTable.display tbody tr.odd.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_2 {
+ background-color: #a8b5cf;
+}
+table.dataTable.display tbody tr.odd.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_3 {
+ background-color: #a9b7d1;
+}
+table.dataTable.display tbody tr.even > .sorting_1, table.dataTable.order-column.stripe tbody tr.even > .sorting_1 {
+ background-color: #fafafa;
+}
+table.dataTable.display tbody tr.even > .sorting_2, table.dataTable.order-column.stripe tbody tr.even > .sorting_2 {
+ background-color: #fcfcfc;
+}
+table.dataTable.display tbody tr.even > .sorting_3, table.dataTable.order-column.stripe tbody tr.even > .sorting_3 {
+ background-color: #fefefe;
+}
+table.dataTable.display tbody tr.even.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_1 {
+ background-color: #acbad5;
+}
+table.dataTable.display tbody tr.even.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_2 {
+ background-color: #aebcd6;
+}
+table.dataTable.display tbody tr.even.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_3 {
+ background-color: #afbdd8;
+}
+table.dataTable.display tbody tr:hover > .sorting_1, table.dataTable.order-column.hover tbody tr:hover > .sorting_1 {
+ background-color: #eaeaea;
+}
+table.dataTable.display tbody tr:hover > .sorting_2, table.dataTable.order-column.hover tbody tr:hover > .sorting_2 {
+ background-color: #ececec;
+}
+table.dataTable.display tbody tr:hover > .sorting_3, table.dataTable.order-column.hover tbody tr:hover > .sorting_3 {
+ background-color: #efefef;
+}
+table.dataTable.display tbody tr:hover.selected > .sorting_1, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_1 {
+ background-color: #a2aec7;
+}
+table.dataTable.display tbody tr:hover.selected > .sorting_2, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_2 {
+ background-color: #a3b0c9;
+}
+table.dataTable.display tbody tr:hover.selected > .sorting_3, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_3 {
+ background-color: #a5b2cb;
+}
+table.dataTable.no-footer {
+ border-bottom: 1px solid #111;
+}
+table.dataTable.nowrap th, table.dataTable.nowrap td {
+ white-space: nowrap;
+}
+table.dataTable.compact thead th,
+table.dataTable.compact thead td {
+ padding: 4px 17px 4px 4px;
+}
+table.dataTable.compact tfoot th,
+table.dataTable.compact tfoot td {
+ padding: 4px;
+}
+table.dataTable.compact tbody th,
+table.dataTable.compact tbody td {
+ padding: 4px;
+}
+table.dataTable th.dt-left,
+table.dataTable td.dt-left {
+ text-align: left;
+}
+table.dataTable th.dt-center,
+table.dataTable td.dt-center,
+table.dataTable td.dataTables_empty {
+ text-align: center;
+}
+table.dataTable th.dt-right,
+table.dataTable td.dt-right {
+ text-align: right;
+}
+table.dataTable th.dt-justify,
+table.dataTable td.dt-justify {
+ text-align: justify;
+}
+table.dataTable th.dt-nowrap,
+table.dataTable td.dt-nowrap {
+ white-space: nowrap;
+}
+table.dataTable thead th.dt-head-left,
+table.dataTable thead td.dt-head-left,
+table.dataTable tfoot th.dt-head-left,
+table.dataTable tfoot td.dt-head-left {
+ text-align: left;
+}
+table.dataTable thead th.dt-head-center,
+table.dataTable thead td.dt-head-center,
+table.dataTable tfoot th.dt-head-center,
+table.dataTable tfoot td.dt-head-center {
+ text-align: center;
+}
+table.dataTable thead th.dt-head-right,
+table.dataTable thead td.dt-head-right,
+table.dataTable tfoot th.dt-head-right,
+table.dataTable tfoot td.dt-head-right {
+ text-align: right;
+}
+table.dataTable thead th.dt-head-justify,
+table.dataTable thead td.dt-head-justify,
+table.dataTable tfoot th.dt-head-justify,
+table.dataTable tfoot td.dt-head-justify {
+ text-align: justify;
+}
+table.dataTable thead th.dt-head-nowrap,
+table.dataTable thead td.dt-head-nowrap,
+table.dataTable tfoot th.dt-head-nowrap,
+table.dataTable tfoot td.dt-head-nowrap {
+ white-space: nowrap;
+}
+table.dataTable tbody th.dt-body-left,
+table.dataTable tbody td.dt-body-left {
+ text-align: left;
+}
+table.dataTable tbody th.dt-body-center,
+table.dataTable tbody td.dt-body-center {
+ text-align: center;
+}
+table.dataTable tbody th.dt-body-right,
+table.dataTable tbody td.dt-body-right {
+ text-align: right;
+}
+table.dataTable tbody th.dt-body-justify,
+table.dataTable tbody td.dt-body-justify {
+ text-align: justify;
+}
+table.dataTable tbody th.dt-body-nowrap,
+table.dataTable tbody td.dt-body-nowrap {
+ white-space: nowrap;
+}
+
+table.dataTable,
+table.dataTable th,
+table.dataTable td {
+ box-sizing: content-box;
+}
+
+/*
+ * Control feature layout
+ */
+.dataTables_wrapper {
+ position: relative;
+ clear: both;
+ *zoom: 1;
+ zoom: 1;
+}
+.dataTables_wrapper .dataTables_length {
+ float: left;
+}
+.dataTables_wrapper .dataTables_filter {
+ float: right;
+ text-align: right;
+}
+.dataTables_wrapper .dataTables_filter input {
+ margin-left: 0.5em;
+}
+.dataTables_wrapper .dataTables_info {
+ clear: both;
+ float: left;
+ padding-top: 0.755em;
+}
+.dataTables_wrapper .dataTables_paginate {
+ float: right;
+ text-align: right;
+ padding-top: 0.25em;
+}
+.dataTables_wrapper .dataTables_paginate .paginate_button {
+ box-sizing: border-box;
+ display: inline-block;
+ min-width: 1.5em;
+ padding: 0.5em 1em;
+ margin-left: 2px;
+ text-align: center;
+ text-decoration: none !important;
+ cursor: pointer;
+ *cursor: hand;
+ color: #333 !important;
+ border: 1px solid transparent;
+ border-radius: 2px;
+}
+.dataTables_wrapper .dataTables_paginate .paginate_button.current, .dataTables_wrapper .dataTables_paginate .paginate_button.current:hover {
+ color: #333 !important;
+ border: 1px solid #979797;
+ background-color: white;
+ background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, white), color-stop(100%, #dcdcdc));
+ /* Chrome,Safari4+ */
+ background: -webkit-linear-gradient(top, white 0%, #dcdcdc 100%);
+ /* Chrome10+,Safari5.1+ */
+ background: -moz-linear-gradient(top, white 0%, #dcdcdc 100%);
+ /* FF3.6+ */
+ background: -ms-linear-gradient(top, white 0%, #dcdcdc 100%);
+ /* IE10+ */
+ background: -o-linear-gradient(top, white 0%, #dcdcdc 100%);
+ /* Opera 11.10+ */
+ background: linear-gradient(to bottom, white 0%, #dcdcdc 100%);
+ /* W3C */
+}
+.dataTables_wrapper .dataTables_paginate .paginate_button.disabled, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active {
+ cursor: default;
+ color: #666 !important;
+ border: 1px solid transparent;
+ background: transparent;
+ box-shadow: none;
+}
+.dataTables_wrapper .dataTables_paginate .paginate_button:hover {
+ color: white !important;
+ border: 1px solid #111;
+ background-color: #585858;
+ background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111));
+ /* Chrome,Safari4+ */
+ background: -webkit-linear-gradient(top, #585858 0%, #111 100%);
+ /* Chrome10+,Safari5.1+ */
+ background: -moz-linear-gradient(top, #585858 0%, #111 100%);
+ /* FF3.6+ */
+ background: -ms-linear-gradient(top, #585858 0%, #111 100%);
+ /* IE10+ */
+ background: -o-linear-gradient(top, #585858 0%, #111 100%);
+ /* Opera 11.10+ */
+ background: linear-gradient(to bottom, #585858 0%, #111 100%);
+ /* W3C */
+}
+.dataTables_wrapper .dataTables_paginate .paginate_button:active {
+ outline: none;
+ background-color: #2b2b2b;
+ background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c));
+ /* Chrome,Safari4+ */
+ background: -webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
+ /* Chrome10+,Safari5.1+ */
+ background: -moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
+ /* FF3.6+ */
+ background: -ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
+ /* IE10+ */
+ background: -o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
+ /* Opera 11.10+ */
+ background: linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%);
+ /* W3C */
+ box-shadow: inset 0 0 3px #111;
+}
+.dataTables_wrapper .dataTables_paginate .ellipsis {
+ padding: 0 1em;
+}
+.dataTables_wrapper .dataTables_processing {
+ position: absolute;
+ top: 50%;
+ left: 50%;
+ width: 100%;
+ height: 40px;
+ margin-left: -50%;
+ margin-top: -25px;
+ padding-top: 20px;
+ text-align: center;
+ font-size: 1.2em;
+ background-color: white;
+ background: -webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255, 255, 255, 0)), color-stop(25%, rgba(255, 255, 255, 0.9)), color-stop(75%, rgba(255, 255, 255, 0.9)), color-stop(100%, rgba(255, 255, 255, 0)));
+ background: -webkit-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
+ background: -moz-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
+ background: -ms-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
+ background: -o-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
+ background: linear-gradient(to right, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
+}
+.dataTables_wrapper .dataTables_length,
+.dataTables_wrapper .dataTables_filter,
+.dataTables_wrapper .dataTables_info,
+.dataTables_wrapper .dataTables_processing,
+.dataTables_wrapper .dataTables_paginate {
+ color: #333;
+}
+.dataTables_wrapper .dataTables_scroll {
+ clear: both;
+}
+.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody {
+ *margin-top: -1px;
+ -webkit-overflow-scrolling: touch;
+}
+.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > thead > tr > th, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > thead > tr > td, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > tbody > tr > th, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > tbody > tr > td {
+ vertical-align: middle;
+}
+.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > thead > tr > th > div.dataTables_sizing,
+.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > thead > tr > td > div.dataTables_sizing, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > tbody > tr > th > div.dataTables_sizing,
+.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > tbody > tr > td > div.dataTables_sizing {
+ height: 0;
+ overflow: hidden;
+ margin: 0 !important;
+ padding: 0 !important;
+}
+.dataTables_wrapper.no-footer .dataTables_scrollBody {
+ border-bottom: 1px solid #111;
+}
+.dataTables_wrapper.no-footer div.dataTables_scrollHead table.dataTable,
+.dataTables_wrapper.no-footer div.dataTables_scrollBody > table {
+ border-bottom: none;
+}
+.dataTables_wrapper:after {
+ visibility: hidden;
+ display: block;
+ content: "";
+ clear: both;
+ height: 0;
+}
+
+@media screen and (max-width: 767px) {
+ .dataTables_wrapper .dataTables_info,
+ .dataTables_wrapper .dataTables_paginate {
+ float: none;
+ text-align: center;
+ }
+ .dataTables_wrapper .dataTables_paginate {
+ margin-top: 0.5em;
+ }
+}
+@media screen and (max-width: 640px) {
+ .dataTables_wrapper .dataTables_length,
+ .dataTables_wrapper .dataTables_filter {
+ float: none;
+ text-align: center;
+ }
+ .dataTables_wrapper .dataTables_filter {
+ margin-top: 0.5em;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css
new file mode 100644
index 0000000..a81a1ac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css
@@ -0,0 +1,352 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+ * File: demo_table_jui.css
+ * CVS: $Id$
+ * Description: CSS descriptions for DataTables demo pages
+ * Author: Allan Jardine
+ * Created: Tue May 12 06:47:22 BST 2009
+ * Modified: $Date$ by $Author$
+ * Language: CSS
+ * Project: DataTables
+ *
+ * Copyright 2009 Allan Jardine. All Rights Reserved.
+ *
+ * ***************************************************************************
+ * DESCRIPTION
+ *
+ * The styles given here are suitable for the demos that are used with the standard DataTables
+ * distribution (see www.datatables.net). You will most likely wish to modify these styles to
+ * meet the layout requirements of your site.
+ *
+ * Common issues:
+ * 'full_numbers' pagination - I use an extra selector on the body tag to ensure that there is
+ * no conflict between the two pagination types. If you want to use full_numbers pagination
+ * ensure that you either have "example_alt_pagination" as a body class name, or better yet,
+ * modify that selector.
+ * Note that the path used for Images is relative. All images are by default located in
+ * ../images/ - relative to this CSS file.
+ */
+
+/*
+ * jQuery UI specific styling
+ */
+
+.paging_two_button .ui-button {
+ float: left;
+ cursor: pointer;
+ *cursor: hand;
+}
+
+.paging_full_numbers .ui-button {
+ padding: 2px 6px;
+ margin: 0;
+ cursor: pointer;
+ *cursor: hand;
+}
+
+.ui-buttonset .ui-button {
+ margin-right: -0.1em !important;
+}
+
+.paging_full_numbers {
+ width: 350px !important;
+}
+
+.ui-toolbar {
+ padding: 5px;
+}
+
+.dataTables_paginate {
+ width: auto;
+}
+
+.dataTables_info {
+ padding-top: 3px;
+}
+
+table.display thead th {
+ padding: 3px 0px 3px 10px;
+ cursor: pointer;
+ *cursor: hand;
+}
+
+div.dataTables_wrapper .ui-widget-header {
+ font-weight: normal;
+}
+
+/*
+ * Sort arrow icon positioning
+ */
+table.display thead th div.DataTables_sort_wrapper {
+ position: relative;
+ padding-right: 20px;
+ padding-right: 20px;
+}
+
+table.display thead th div.DataTables_sort_wrapper span {
+ position: absolute;
+ top: 50%;
+ margin-top: -8px;
+ right: 0;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ *
+ * Everything below this line is the same as demo_table.css. This file is
+ * required for 'cleanliness' of the markup
+ *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables features
+ */
+
+.dataTables_wrapper {
+ position: relative;
+ min-height: 35px;
+ _height: 35px;
+ clear: both;
+}
+
+.dataTables_processing {
+ position: absolute;
+ top: 0px;
+ left: 50%;
+ width: 250px;
+ margin-left: -125px;
+ border: 1px solid #ddd;
+ text-align: center;
+ color: #999;
+ font-size: 11px;
+ padding: 2px 0;
+}
+
+.dataTables_length {
+ width: 40%;
+ float: left;
+}
+
+.dataTables_filter {
+ width: 50%;
+ float: right;
+ text-align: right;
+}
+
+.dataTables_info {
+ width: 50%;
+ float: left;
+}
+
+.dataTables_paginate {
+ float: right;
+ text-align: right;
+}
+
+/* Pagination nested */
+.paginate_disabled_previous,
+.paginate_enabled_previous,
+.paginate_disabled_next,
+.paginate_enabled_next {
+ height: 19px;
+ width: 19px;
+ margin-left: 3px;
+ float: left;
+}
+
+.paginate_disabled_previous {
+ background-image: url("../images/back_disabled.jpg");
+}
+
+.paginate_enabled_previous {
+ background-image: url("../images/back_enabled.jpg");
+}
+
+.paginate_disabled_next {
+ background-image: url("../images/forward_disabled.jpg");
+}
+
+.paginate_enabled_next {
+ background-image: url("../images/forward_enabled.jpg");
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables display
+ */
+table.display {
+ margin: 0 auto;
+ width: 100%;
+ clear: both;
+ border-collapse: collapse;
+}
+
+table.display tfoot th {
+ padding: 3px 0px 3px 10px;
+ font-weight: bold;
+ font-weight: normal;
+}
+
+table.display tr.heading2 td {
+ border-bottom: 1px solid #aaa;
+}
+
+table.display td {
+ padding: 3px 10px;
+}
+
+table.display td.center {
+ text-align: center;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables sorting
+ */
+
+.sorting_asc {
+ background: url("../images/sort_asc.png") no-repeat center right;
+}
+
+.sorting_desc {
+ background: url("../images/sort_desc.png") no-repeat center right;
+}
+
+.sorting {
+ background: url("../images/sort_both.png") no-repeat center right;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Misc
+ */
+.dataTables_scroll {
+ clear: both;
+}
+
+.top,
+.bottom {
+ padding: 15px;
+ background-color: #f5f5f5;
+ border: 1px solid #cccccc;
+}
+
+.top .dataTables_info {
+ float: none;
+}
+
+.clear {
+ clear: both;
+}
+
+.dataTables_empty {
+ text-align: center;
+}
+
+tfoot input {
+ margin: 0.5em 0;
+ width: 100%;
+ color: #444;
+}
+
+tfoot input.search_init {
+ color: #999;
+}
+
+td.group {
+ background-color: #d1cfd0;
+ border-bottom: 2px solid #a19b9e;
+ border-top: 2px solid #a19b9e;
+}
+
+td.details {
+ background-color: #d1cfd0;
+ border: 2px solid #a19b9e;
+}
+
+.example_alt_pagination div.dataTables_info {
+ width: 40%;
+}
+
+.paging_full_numbers span.paginate_button,
+.paging_full_numbers span.paginate_active {
+ border: 1px solid #aaa;
+ -webkit-border-radius: 5px;
+ -moz-border-radius: 5px;
+ padding: 2px 5px;
+ margin: 0 3px;
+ cursor: pointer;
+ *cursor: hand;
+}
+
+.paging_full_numbers span.paginate_button {
+ background-color: #ddd;
+}
+
+.paging_full_numbers span.paginate_button:hover {
+ background-color: #ccc;
+}
+
+.paging_full_numbers span.paginate_active {
+ background-color: #99b3ff;
+}
+
+table.display tr.even.row_selected td {
+ background-color: #b0bed9;
+}
+
+table.display tr.odd.row_selected td {
+ background-color: #9fafd1;
+}
+
+/* Striping */
+tr.odd {
+ background: rgba(255, 255, 255, 0.1);
+}
+tr.even {
+ background: rgba(0, 0, 255, 0.05);
+}
+
+/*
+ * Sorting classes for columns
+ */
+tr.odd td.sorting_1 {
+ background: rgba(0, 0, 0, 0.03);
+}
+tr.odd td.sorting_2 {
+ background: rgba(0, 0, 0, 0.02);
+}
+tr.odd td.sorting_3 {
+ background: rgba(0, 0, 0, 0.02);
+}
+tr.even td.sorting_1 {
+ background: rgba(0, 0, 0, 0.08);
+}
+tr.even td.sorting_2 {
+ background: rgba(0, 0, 0, 0.06);
+}
+tr.even td.sorting_3 {
+ background: rgba(0, 0, 0, 0.06);
+}
+
+.css_left {
+ position: relative;
+ float: left;
+}
+.css_right {
+ position: relative;
+ float: right;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/Sorting icons.psd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/Sorting icons.psd b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/Sorting icons.psd
new file mode 100644
index 0000000..53b2e06
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/Sorting icons.psd differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_disabled.jpg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_disabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_disabled.jpg
new file mode 100644
index 0000000..1e73a54
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_disabled.jpg differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_enabled.jpg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_enabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_enabled.jpg
new file mode 100644
index 0000000..a6d764c
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_enabled.jpg differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/favicon.ico
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/favicon.ico b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/favicon.ico
new file mode 100644
index 0000000..6eeaa2a
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/favicon.ico differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_disabled.jpg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_disabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_disabled.jpg
new file mode 100644
index 0000000..28a9dc5
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_disabled.jpg differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_enabled.jpg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_enabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_enabled.jpg
new file mode 100644
index 0000000..598c075
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_enabled.jpg differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc.png
new file mode 100644
index 0000000..a56d0e2
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc.png differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc_disabled.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc_disabled.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc_disabled.png
new file mode 100644
index 0000000..b7e621e
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc_disabled.png differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_both.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_both.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_both.png
new file mode 100644
index 0000000..839ac4b
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_both.png differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc.png
new file mode 100644
index 0000000..90b2951
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc.png differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36012b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc_disabled.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc_disabled.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc_disabled.png
new file mode 100644
index 0000000..2409653
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc_disabled.png differ
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDDS-799. Avoid ByteString to byte
array conversion cost by using ByteBuffer in Datanode. Contributed by Mukul
Kumar Singh.
Posted by ae...@apache.org.
HDDS-799. Avoid ByteString to byte array conversion cost by using ByteBuffer in Datanode. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/942693bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/942693bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/942693bd
Branch: refs/heads/HDDS-4
Commit: 942693bddd5fba51b85a5f677e3496a41817cff3
Parents: c8ca174
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Mon Nov 5 23:43:22 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Mon Nov 5 23:43:22 2018 +0530
----------------------------------------------------------------------
.../container/keyvalue/KeyValueHandler.java | 11 +++---
.../container/keyvalue/helpers/ChunkUtils.java | 28 ++++++++-------
.../keyvalue/impl/ChunkManagerImpl.java | 2 +-
.../keyvalue/interfaces/ChunkManager.java | 3 +-
.../keyvalue/TestChunkManagerImpl.java | 37 ++++++++++----------
.../common/impl/TestContainerPersistence.java | 28 ++++++++++-----
6 files changed, 62 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/942693bd/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 4cb23ed..1271d99 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.keyvalue;
import java.io.FileInputStream;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -76,7 +77,7 @@ import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import static org.apache.hadoop.hdds.HddsConfigKeys
.HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.*;
@@ -652,10 +653,10 @@ public class KeyValueHandler extends Handler {
ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(chunkInfoProto);
Preconditions.checkNotNull(chunkInfo);
- byte[] data = null;
+ ByteBuffer data = null;
if (request.getWriteChunk().getStage() == Stage.WRITE_DATA ||
request.getWriteChunk().getStage() == Stage.COMBINED) {
- data = request.getWriteChunk().getData().toByteArray();
+ data = request.getWriteChunk().getData().asReadOnlyByteBuffer();
}
chunkManager.writeChunk(kvContainer, blockID, chunkInfo, data,
@@ -713,7 +714,7 @@ public class KeyValueHandler extends Handler {
ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(
putSmallFileReq.getChunkInfo());
Preconditions.checkNotNull(chunkInfo);
- byte[] data = putSmallFileReq.getData().toByteArray();
+ ByteBuffer data = putSmallFileReq.getData().asReadOnlyByteBuffer();
// chunks will be committed as a part of handling putSmallFile
// here. There is no need to maintain this info in openContainerBlockMap.
chunkManager.writeChunk(
@@ -724,7 +725,7 @@ public class KeyValueHandler extends Handler {
blockData.setChunks(chunks);
// TODO: add bcsId as a part of putSmallFile transaction
blockManager.putBlock(kvContainer, blockData);
- metrics.incContainerBytesStats(Type.PutSmallFile, data.length);
+ metrics.incContainerBytesStats(Type.PutSmallFile, data.capacity());
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/942693bd/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 20598d9..718f5de 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -73,15 +73,15 @@ public final class ChunkUtils {
* @throws StorageContainerException
*/
public static void writeData(File chunkFile, ChunkInfo chunkInfo,
- byte[] data, VolumeIOStats volumeIOStats) throws
- StorageContainerException, ExecutionException, InterruptedException,
- NoSuchAlgorithmException {
-
+ ByteBuffer data, VolumeIOStats volumeIOStats)
+ throws StorageContainerException, ExecutionException,
+ InterruptedException, NoSuchAlgorithmException {
+ int bufferSize = data.capacity();
Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
- if (data.length != chunkInfo.getLen()) {
+ if (bufferSize != chunkInfo.getLen()) {
String err = String.format("data array does not match the length " +
"specified. DataLen: %d Byte Array: %d",
- chunkInfo.getLen(), data.length);
+ chunkInfo.getLen(), bufferSize);
log.error(err);
throw new StorageContainerException(err, INVALID_WRITE_SIZE);
}
@@ -103,16 +103,16 @@ public final class ChunkUtils {
StandardOpenOption.SPARSE,
StandardOpenOption.SYNC);
lock = file.lock().get();
- int size = file.write(ByteBuffer.wrap(data), chunkInfo.getOffset()).get();
+ int size = file.write(data, chunkInfo.getOffset()).get();
// Increment volumeIO stats here.
volumeIOStats.incWriteTime(Time.monotonicNow() - writeTimeStart);
volumeIOStats.incWriteOpCount();
volumeIOStats.incWriteBytes(size);
- if (size != data.length) {
+ if (size != bufferSize) {
log.error("Invalid write size found. Size:{} Expected: {} ", size,
- data.length);
+ bufferSize);
throw new StorageContainerException("Invalid write size found. " +
- "Size: " + size + " Expected: " + data.length, INVALID_WRITE_SIZE);
+ "Size: " + size + " Expected: " + bufferSize, INVALID_WRITE_SIZE);
}
} catch (StorageContainerException ex) {
throw ex;
@@ -183,7 +183,8 @@ public final class ChunkUtils {
volumeIOStats.incReadOpCount();
volumeIOStats.incReadBytes(data.getLen());
if (data.getChecksum() != null && !data.getChecksum().isEmpty()) {
- verifyChecksum(data, buf.array(), log);
+ buf.rewind();
+ verifyChecksum(data, buf, log);
}
return buf;
} catch (IOException e) {
@@ -211,10 +212,11 @@ public final class ChunkUtils {
* @throws NoSuchAlgorithmException
* @throws StorageContainerException
*/
- private static void verifyChecksum(ChunkInfo chunkInfo, byte[] data, Logger
- log) throws NoSuchAlgorithmException, StorageContainerException {
+ private static void verifyChecksum(ChunkInfo chunkInfo, ByteBuffer data,
+ Logger log) throws NoSuchAlgorithmException, StorageContainerException {
MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
sha.update(data);
+ data.rewind();
if (!Hex.encodeHexString(sha.digest()).equals(
chunkInfo.getChecksum())) {
log.error("Checksum mismatch. Provided: {} , computed: {}",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/942693bd/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index 6fd8d5f..c630e19 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -66,7 +66,7 @@ public class ChunkManagerImpl implements ChunkManager {
* @throws StorageContainerException
*/
public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
- byte[] data, ContainerProtos.Stage stage)
+ ByteBuffer data, ContainerProtos.Stage stage)
throws StorageContainerException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/942693bd/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
index 7134be1..3b06585 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import java.nio.ByteBuffer;
/**
* Chunk Manager allows read, write, delete and listing of chunks in
@@ -41,7 +42,7 @@ public interface ChunkManager {
* @throws StorageContainerException
*/
void writeChunk(Container container, BlockID blockID, ChunkInfo info,
- byte[] data, ContainerProtos.Stage stage)
+ ByteBuffer data, ContainerProtos.Stage stage)
throws StorageContainerException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/942693bd/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index 3c0876b..9e3edf7 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -39,6 +39,7 @@ import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import java.io.File;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.UUID;
@@ -109,8 +110,8 @@ public class TestChunkManagerImpl {
// As no chunks are written to the volume writeBytes should be 0
checkWriteIOStats(0, 0);
- chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data,
- ContainerProtos.Stage.WRITE_DATA);
+ chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
+ ByteBuffer.wrap(data), ContainerProtos.Stage.WRITE_DATA);
// Now a chunk file is being written with Stage WRITE_DATA, so it should
// create a temporary chunk file.
assertTrue(chunksPath.listFiles().length == 1);
@@ -126,8 +127,8 @@ public class TestChunkManagerImpl {
checkWriteIOStats(data.length, 1);
- chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data,
- ContainerProtos.Stage.COMMIT_DATA);
+ chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
+ ByteBuffer.wrap(data), ContainerProtos.Stage.COMMIT_DATA);
checkWriteIOStats(data.length, 1);
@@ -146,8 +147,8 @@ public class TestChunkManagerImpl {
long randomLength = 200L;
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), 0), 0, randomLength);
- chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data,
- ContainerProtos.Stage.WRITE_DATA);
+ chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
+ ByteBuffer.wrap(data), ContainerProtos.Stage.WRITE_DATA);
fail("testWriteChunkIncorrectLength failed");
} catch (StorageContainerException ex) {
// As we got an exception, writeBytes should be 0.
@@ -167,8 +168,8 @@ public class TestChunkManagerImpl {
// Initially chunks folder should be empty.
assertTrue(chunksPath.listFiles().length == 0);
checkWriteIOStats(0, 0);
- chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data,
- ContainerProtos.Stage.COMBINED);
+ chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
+ ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
// Now a chunk file is being written with Stage COMBINED_DATA, so it should
// create a chunk file.
assertTrue(chunksPath.listFiles().length == 1);
@@ -180,8 +181,8 @@ public class TestChunkManagerImpl {
@Test
public void testReadChunk() throws Exception {
checkWriteIOStats(0, 0);
- chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data,
- ContainerProtos.Stage.COMBINED);
+ chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
+ ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
checkWriteIOStats(data.length, 1);
checkReadIOStats(0, 0);
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
@@ -194,8 +195,8 @@ public class TestChunkManagerImpl {
@Test
public void testDeleteChunk() throws Exception {
File chunksPath = new File(keyValueContainerData.getChunksPath());
- chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data,
- ContainerProtos.Stage.COMBINED);
+ chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
+ ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
assertTrue(chunksPath.listFiles().length == 1);
chunkManager.deleteChunk(keyValueContainer, blockID, chunkInfo);
assertTrue(chunksPath.listFiles().length == 0);
@@ -204,8 +205,8 @@ public class TestChunkManagerImpl {
@Test
public void testDeleteChunkUnsupportedRequest() throws Exception {
try {
- chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data,
- ContainerProtos.Stage.COMBINED);
+ chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
+ ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
long randomLength = 200L;
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), 0), 0, randomLength);
@@ -224,8 +225,8 @@ public class TestChunkManagerImpl {
.getLocalID(), 0), 0, data.length);
//Setting checksum to some value.
chunkInfo.setChecksum("some garbage");
- chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data,
- ContainerProtos.Stage.COMBINED);
+ chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
+ ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
fail("testWriteChunkChecksumMismatch failed");
} catch (StorageContainerException ex) {
GenericTestUtils.assertExceptionContains("Checksum mismatch.", ex);
@@ -252,8 +253,8 @@ public class TestChunkManagerImpl {
for (int i=0; i<100; i++) {
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), i), 0, data.length);
- chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data,
- ContainerProtos.Stage.COMBINED);
+ chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
+ ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
}
checkWriteIOStats(data.length*100, 100);
assertTrue(hddsVolume.getVolumeIOStats().getWriteTime() > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/942693bd/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index c2941ed..f6c498a 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -59,6 +59,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -330,7 +331,8 @@ public class TestContainerPersistence {
blockID.getLocalID(), 0, 0, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
- chunkManager.writeChunk(container, blockID, info, data, COMBINED);
+ chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
+ COMBINED);
return info;
}
@@ -371,7 +373,8 @@ public class TestContainerPersistence {
ChunkInfo info = getChunk(blockID.getLocalID(), x, 0, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
- chunkManager.writeChunk(container, blockID, info, data, COMBINED);
+ chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
+ COMBINED);
String fileName = String.format("%s.data.%d", blockID.getLocalID(), x);
fileHashMap.put(fileName, info);
}
@@ -431,7 +434,8 @@ public class TestContainerPersistence {
blockID.getLocalID(), 0, 0, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
- chunkManager.writeChunk(container, blockID, info, data, COMBINED);
+ chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
+ COMBINED);
byte[] readData = chunkManager.readChunk(container, blockID, info);
assertTrue(Arrays.equals(data, readData));
@@ -463,11 +467,14 @@ public class TestContainerPersistence {
blockID.getLocalID(), 0, 0, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
- chunkManager.writeChunk(container, blockID, info, data, COMBINED);
- chunkManager.writeChunk(container, blockID, info, data, COMBINED);
+ chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
+ COMBINED);
+ chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
+ COMBINED);
// With the overwrite flag it should work now.
info.addMetadata(OzoneConsts.CHUNK_OVERWRITE, "true");
- chunkManager.writeChunk(container, blockID, info, data, COMBINED);
+ chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
+ COMBINED);
long bytesUsed = container.getContainerData().getBytesUsed();
Assert.assertEquals(datalen, bytesUsed);
@@ -501,7 +508,8 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
oldSha.update(data);
setDataChecksum(info, data);
- chunkManager.writeChunk(container, blockID, info, data, COMBINED);
+ chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
+ COMBINED);
}
// Request to read the whole data in a single go.
@@ -532,7 +540,8 @@ public class TestContainerPersistence {
blockID.getLocalID(), 0, 0, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
- chunkManager.writeChunk(container, blockID, info, data, COMBINED);
+ chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
+ COMBINED);
chunkManager.deleteChunk(container, blockID, info);
exception.expect(StorageContainerException.class);
exception.expectMessage("Unable to find the chunk file.");
@@ -646,7 +655,8 @@ public class TestContainerPersistence {
info = getChunk(blockID.getLocalID(), x, x * datalen, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
- chunkManager.writeChunk(container, blockID, info, data, COMBINED);
+ chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
+ COMBINED);
totalSize += datalen;
chunkList.add(info);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDDS-762. Fix unit test failure
for TestContainerSQLCli & TestSCMMetrics. Contributed by Mukul Kumar Singh.
Posted by ae...@apache.org.
HDDS-762. Fix unit test failure for TestContainerSQLCli & TestSCMMetrics.
Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e33b61f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e33b61f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e33b61f3
Branch: refs/heads/HDDS-4
Commit: e33b61f3351c09b00717f6eef32ff7d24345d06e
Parents: f747f5b
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 30 19:16:52 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 30 19:16:52 2018 -0700
----------------------------------------------------------------------
hadoop-hdds/pom.xml | 2 +-
.../common/transport/server/ratis/TestCSMMetrics.java | 10 +++++++---
hadoop-ozone/pom.xml | 2 +-
.../org/apache/hadoop/ozone/scm/TestContainerSQLCli.java | 3 ++-
4 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e33b61f3/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index f960e90..090a537 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -45,7 +45,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<hdds.version>0.4.0-SNAPSHOT</hdds.version>
<!-- Apache Ratis version -->
- <ratis.version>0.3.0-2272086-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-1d2ebee-SNAPSHOT</ratis.version>
<bouncycastle.version>1.60</bouncycastle.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e33b61f3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
index a5a9641..67db7ff 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import static org.apache.ratis.rpc.SupportedRpcType.GRPC;
+import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.util.CheckedBiConsumer;
import java.util.function.BiConsumer;
@@ -104,7 +105,8 @@ public class TestCSMMetrics {
client.connect();
// Before Read Chunk/Write Chunk
- MetricsRecordBuilder metric = getMetrics(CSMMetrics.SOURCE_NAME);
+ MetricsRecordBuilder metric = getMetrics(CSMMetrics.SOURCE_NAME +
+ RaftGroupId.valueOf(pipeline.getId().getId()).toString());
assertCounter("NumWriteStateMachineOps", 0L, metric);
assertCounter("NumReadStateMachineOps", 0L, metric);
assertCounter("NumApplyTransactionOps", 0L, metric);
@@ -120,7 +122,8 @@ public class TestCSMMetrics {
Assert.assertEquals(ContainerProtos.Result.SUCCESS,
response.getResult());
- metric = getMetrics(CSMMetrics.SOURCE_NAME);
+ metric = getMetrics(CSMMetrics.SOURCE_NAME +
+ RaftGroupId.valueOf(pipeline.getId().getId()).toString());
assertCounter("NumWriteStateMachineOps", 1L, metric);
assertCounter("NumApplyTransactionOps", 1L, metric);
@@ -132,7 +135,8 @@ public class TestCSMMetrics {
Assert.assertEquals(ContainerProtos.Result.SUCCESS,
response.getResult());
- metric = getMetrics(CSMMetrics.SOURCE_NAME);
+ metric = getMetrics(CSMMetrics.SOURCE_NAME +
+ RaftGroupId.valueOf(pipeline.getId().getId()).toString());
assertCounter("NumReadStateMachineOps", 1L, metric);
assertCounter("NumApplyTransactionOps", 1L, metric);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e33b61f3/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 2fcffab..33af31b 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -33,7 +33,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<hadoop.version>3.2.1-SNAPSHOT</hadoop.version>
<hdds.version>0.4.0-SNAPSHOT</hdds.version>
<ozone.version>0.4.0-SNAPSHOT</ozone.version>
- <ratis.version>0.3.0-2272086-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-1d2ebee-SNAPSHOT</ratis.version>
<bouncycastle.version>1.60</bouncycastle.version>
<ozone.release>Badlands</ozone.release>
<declared.ozone.version>${ozone.version}</declared.ozone.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e33b61f3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index 9117838..054e668 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -122,7 +123,7 @@ public class TestContainerSQLCli {
eventQueue = new EventQueue();
nodeManager = cluster.getStorageContainerManager().getScmNodeManager();
PipelineManager pipelineManager =
- cluster.getStorageContainerManager().getPipelineManager();
+ new SCMPipelineManager(conf, nodeManager, eventQueue);
containerManager = new SCMContainerManager(conf, nodeManager,
pipelineManager, eventQueue);
blockManager = new BlockManagerImpl(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HADOOP-15895. [JDK9+] Add missing
javax.annotation-api dependency to hadoop-yarn-csi.
Posted by ae...@apache.org.
HADOOP-15895. [JDK9+] Add missing javax.annotation-api dependency to hadoop-yarn-csi.
Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d174b916
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d174b916
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d174b916
Branch: refs/heads/HDDS-4
Commit: d174b916352ffe701058f3bdae433c3f24eb37c2
Parents: e0ac308
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 1 10:11:21 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 1 12:56:20 2018 +0000
----------------------------------------------------------------------
hadoop-project/pom.xml | 5 +++++
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml | 5 +++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d174b916/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 2247109..937f21f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1474,6 +1474,11 @@
<artifactId>javax.activation-api</artifactId>
<version>1.2.0</version>
</dependency>
+ <dependency>
+ <groupId>javax.annotation</groupId>
+ <artifactId>javax.annotation-api</artifactId>
+ <version>1.3.2</version>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d174b916/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
index b58b0fe..41f5098 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
@@ -83,6 +83,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>javax.annotation</groupId>
+ <artifactId>javax.annotation-api</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDDS-794. addendum patch to fix
compilation failure. Contributed by Shashikant Banerjee.
Posted by ae...@apache.org.
HDDS-794. addendum patch to fix compilation failure. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50f40e05
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50f40e05
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50f40e05
Branch: refs/heads/HDDS-4
Commit: 50f40e0536f38517aa33e8859f299bcf19f2f319
Parents: 5ddefdd
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Tue Nov 6 00:20:57 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Tue Nov 6 00:20:57 2018 +0530
----------------------------------------------------------------------
.../apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50f40e05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 8f9d589..dc44dc5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -139,7 +139,7 @@ public final class ChunkUtils {
}
}
log.debug("Write Chunk completed for chunkFile: {}, size {}", chunkFile,
- data.length);
+ bufferSize);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HADOOP-15886. Fix findbugs
warnings in RegistryDNS.java.
Posted by ae...@apache.org.
HADOOP-15886. Fix findbugs warnings in RegistryDNS.java.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f747f5b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f747f5b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f747f5b0
Branch: refs/heads/HDDS-4
Commit: f747f5b06cb0da59c7c20b9f0e46d3eec9622eed
Parents: 277a3d8
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Oct 30 11:43:36 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 31 10:01:31 2018 +0900
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 33 ++++++++++++++++++++
hadoop-common-project/hadoop-registry/pom.xml | 10 ++++++
.../dev-support/findbugs-exclude.xml | 16 ----------
3 files changed, 43 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f747f5b0/hadoop-common-project/hadoop-registry/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-registry/dev-support/findbugs-exclude.xml b/hadoop-common-project/hadoop-registry/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..dc7b139
--- /dev/null
+++ b/hadoop-common-project/hadoop-registry/dev-support/findbugs-exclude.xml
@@ -0,0 +1,33 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <Match>
+ <Class name="org.apache.hadoop.registry.server.dns.RegistryDNS" />
+ <Method name="addNIOTCP" />
+ <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.registry.server.dns.RegistryDNS" />
+ <Method name="addNIOUDP" />
+ <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.registry.server.dns.RegistryDNS" />
+ <Method name="serveNIOTCP" />
+ <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+ </Match>
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f747f5b0/hadoop-common-project/hadoop-registry/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-registry/pom.xml b/hadoop-common-project/hadoop-registry/pom.xml
index ef9f3ef..7ca1c9e 100644
--- a/hadoop-common-project/hadoop-registry/pom.xml
+++ b/hadoop-common-project/hadoop-registry/pom.xml
@@ -163,6 +163,16 @@
</resources>
<plugins>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <findbugsXmlOutput>true</findbugsXmlOutput>
+ <xmlOutput>true</xmlOutput>
+ <excludeFilterFile>${project.basedir}/dev-support/findbugs-exclude.xml</excludeFilterFile>
+ <effort>Max</effort>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f747f5b0/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 216c3bd..dd42129 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -639,22 +639,6 @@
<Bug pattern="MS_EXPOSE_REP" />
</Match>
- <Match>
- <Class name="org.apache.hadoop.registry.server.dns.RegistryDNS" />
- <Method name="addNIOTCP" />
- <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
- </Match>
- <Match>
- <Class name="org.apache.hadoop.registry.server.dns.RegistryDNS" />
- <Method name="addNIOUDP" />
- <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
- </Match>
- <Match>
- <Class name="org.apache.hadoop.registry.server.dns.RegistryDNS" />
- <Method name="serveNIOTCP" />
- <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
- </Match>
-
<!-- EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC -->
<Match>
<Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice" />
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HDFS-14042. Fix NPE when PROVIDED
storage is missing. Contributed by Virajith Jalaparti.
Posted by ae...@apache.org.
HDFS-14042. Fix NPE when PROVIDED storage is missing. Contributed by Virajith Jalaparti.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3f5e7ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3f5e7ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3f5e7ad
Branch: refs/heads/HDDS-4
Commit: f3f5e7ad005a88afad6fa09602073eaa450e21ed
Parents: 50f40e0
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Mon Nov 5 11:02:31 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Mon Nov 5 11:02:31 2018 -0800
----------------------------------------------------------------------
.../hdfs/server/blockmanagement/BlockManager.java | 13 ++++++++++++-
.../server/blockmanagement/DatanodeDescriptor.java | 4 ++--
.../hdfs/server/blockmanagement/HeartbeatManager.java | 2 +-
.../hdfs/server/datanode/TestDataNodeLifeline.java | 5 +++++
4 files changed, 20 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f5e7ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index d74b523..a5fb0b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2447,7 +2447,7 @@ public class BlockManager implements BlockStatsMXBean {
return providedStorageMap.getCapacity();
}
- public void updateHeartbeat(DatanodeDescriptor node, StorageReport[] reports,
+ void updateHeartbeat(DatanodeDescriptor node, StorageReport[] reports,
long cacheCapacity, long cacheUsed, int xceiverCount, int failedVolumes,
VolumeFailureSummary volumeFailureSummary) {
@@ -2458,6 +2458,17 @@ public class BlockManager implements BlockStatsMXBean {
failedVolumes, volumeFailureSummary);
}
+ void updateHeartbeatState(DatanodeDescriptor node,
+ StorageReport[] reports, long cacheCapacity, long cacheUsed,
+ int xceiverCount, int failedVolumes,
+ VolumeFailureSummary volumeFailureSummary) {
+ for (StorageReport report: reports) {
+ providedStorageMap.updateStorage(node, report.getStorage());
+ }
+ node.updateHeartbeatState(reports, cacheCapacity, cacheUsed, xceiverCount,
+ failedVolumes, volumeFailureSummary);
+ }
+
/**
* StatefulBlockInfo is used to build the "toUC" list, which is a list of
* updates to the information about under-construction blocks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f5e7ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 12b5c33..6aa2376 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -373,7 +373,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
/**
* Updates stats from datanode heartbeat.
*/
- public void updateHeartbeat(StorageReport[] reports, long cacheCapacity,
+ void updateHeartbeat(StorageReport[] reports, long cacheCapacity,
long cacheUsed, int xceiverCount, int volFailures,
VolumeFailureSummary volumeFailureSummary) {
updateHeartbeatState(reports, cacheCapacity, cacheUsed, xceiverCount,
@@ -384,7 +384,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
/**
* process datanode heartbeat or stats initialization.
*/
- public void updateHeartbeatState(StorageReport[] reports, long cacheCapacity,
+ void updateHeartbeatState(StorageReport[] reports, long cacheCapacity,
long cacheUsed, int xceiverCount, int volFailures,
VolumeFailureSummary volumeFailureSummary) {
updateStorageStats(reports, cacheCapacity, cacheUsed, xceiverCount,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f5e7ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
index 1972a6d..d2c279f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
@@ -251,7 +251,7 @@ class HeartbeatManager implements DatanodeStatistics {
// updateHeartbeat, because we don't want to modify the
// heartbeatedSinceRegistration flag. Arrival of a lifeline message does
// not count as arrival of the first heartbeat.
- node.updateHeartbeatState(reports, cacheCapacity, cacheUsed,
+ blockManager.updateHeartbeatState(node, reports, cacheCapacity, cacheUsed,
xceiverCount, failedVolumes, volumeFailureSummary);
stats.add(node);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f5e7ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java
index 28427bc..14134e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.datanode;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY;
@@ -196,6 +197,10 @@ public class TestDataNodeLifeline {
namesystem.getNumDeadDataNodes());
assertEquals("Expect DataNode not marked stale due to lifeline.", 0,
namesystem.getNumStaleDataNodes());
+ // add a new volume on the next heartbeat
+ cluster.getDataNodes().get(0).reconfigurePropertyImpl(
+ DFS_DATANODE_DATA_DIR_KEY,
+ cluster.getDataDirectory().concat("/data-new"));
}
// Verify that we did in fact call the lifeline RPC.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDDS-797. If DN is started before
SCM, it does not register. Contributed by Hanisha Koneru.
Posted by ae...@apache.org.
HDDS-797. If DN is started before SCM, it does not register. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8ca1747
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8ca1747
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8ca1747
Branch: refs/heads/HDDS-4
Commit: c8ca1747c08d905cdefaa5566dd58d770a6b71bd
Parents: 15df2e7
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Nov 5 09:40:00 2018 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Nov 5 09:40:00 2018 -0800
----------------------------------------------------------------------
.../states/endpoint/VersionEndpointTask.java | 79 +++++++++++---------
.../hadoop/ozone/TestMiniOzoneCluster.java | 52 ++++++++++++-
2 files changed, 94 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8ca1747/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
index 79fa174..2d00da8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
@@ -64,50 +64,57 @@ public class VersionEndpointTask implements
public EndpointStateMachine.EndPointStates call() throws Exception {
rpcEndPoint.lock();
try{
- SCMVersionResponseProto versionResponse =
- rpcEndPoint.getEndPoint().getVersion(null);
- VersionResponse response = VersionResponse.getFromProtobuf(
- versionResponse);
- rpcEndPoint.setVersion(response);
+ if (rpcEndPoint.getState().equals(
+ EndpointStateMachine.EndPointStates.GETVERSION)) {
+ SCMVersionResponseProto versionResponse =
+ rpcEndPoint.getEndPoint().getVersion(null);
+ VersionResponse response = VersionResponse.getFromProtobuf(
+ versionResponse);
+ rpcEndPoint.setVersion(response);
- String scmId = response.getValue(OzoneConsts.SCM_ID);
- String clusterId = response.getValue(OzoneConsts.CLUSTER_ID);
+ String scmId = response.getValue(OzoneConsts.SCM_ID);
+ String clusterId = response.getValue(OzoneConsts.CLUSTER_ID);
- // Check volumes
- VolumeSet volumeSet = ozoneContainer.getVolumeSet();
- volumeSet.writeLock();
- try {
- Map<String, HddsVolume> volumeMap = volumeSet.getVolumeMap();
+ // Check volumes
+ VolumeSet volumeSet = ozoneContainer.getVolumeSet();
+ volumeSet.writeLock();
+ try {
+ Map<String, HddsVolume> volumeMap = volumeSet.getVolumeMap();
- Preconditions.checkNotNull(scmId, "Reply from SCM: scmId cannot be " +
- "null");
- Preconditions.checkNotNull(clusterId, "Reply from SCM: clusterId " +
- "cannot be null");
+ Preconditions.checkNotNull(scmId, "Reply from SCM: scmId cannot be " +
+ "null");
+ Preconditions.checkNotNull(clusterId, "Reply from SCM: clusterId " +
+ "cannot be null");
- // If version file does not exist create version file and also set scmId
- for (Map.Entry<String, HddsVolume> entry : volumeMap.entrySet()) {
- HddsVolume hddsVolume = entry.getValue();
- boolean result = HddsVolumeUtil.checkVolume(hddsVolume, scmId,
- clusterId, LOG);
- if (!result) {
- volumeSet.failVolume(hddsVolume.getHddsRootDir().getPath());
+ // If version file does not exist create version file and also set scmId
+
+ for (Map.Entry<String, HddsVolume> entry : volumeMap.entrySet()) {
+ HddsVolume hddsVolume = entry.getValue();
+ boolean result = HddsVolumeUtil.checkVolume(hddsVolume, scmId,
+ clusterId, LOG);
+ if (!result) {
+ volumeSet.failVolume(hddsVolume.getHddsRootDir().getPath());
+ }
}
+ if (volumeSet.getVolumesList().size() == 0) {
+ // All volumes are in inconsistent state
+ throw new DiskOutOfSpaceException("All configured Volumes are in " +
+ "Inconsistent State");
+ }
+ } finally {
+ volumeSet.writeUnlock();
}
- if (volumeSet.getVolumesList().size() == 0) {
- // All volumes are in inconsistent state
- throw new DiskOutOfSpaceException("All configured Volumes are in " +
- "Inconsistent State");
- }
- } finally {
- volumeSet.writeUnlock();
- }
- ozoneContainer.getDispatcher().setScmId(scmId);
+ ozoneContainer.getDispatcher().setScmId(scmId);
- EndpointStateMachine.EndPointStates nextState =
- rpcEndPoint.getState().getNextState();
- rpcEndPoint.setState(nextState);
- rpcEndPoint.zeroMissedCount();
+ EndpointStateMachine.EndPointStates nextState =
+ rpcEndPoint.getState().getNextState();
+ rpcEndPoint.setState(nextState);
+ rpcEndPoint.zeroMissedCount();
+ } else {
+ LOG.debug("Cannot execute GetVersion task as endpoint state machine " +
+ "is in {} state", rpcEndPoint.getState());
+ }
} catch (DiskOutOfSpaceException ex) {
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.SHUTDOWN);
} catch(IOException ex) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8ca1747/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index 0ca4749..3083660 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -23,9 +23,13 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine
+ .DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine
+ .EndpointStateMachine;
import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.TestUtils;
@@ -214,4 +218,50 @@ public class TestMiniOzoneCluster {
out.write("malformed".getBytes());
out.close();
}
+
+ /**
+ * Test that a DN can register with SCM even if it was started before the SCM.
+ * @throws Exception
+ */
+ @Test (timeout = 300_000)
+ public void testDNstartAfterSCM() throws Exception {
+ // Start a cluster with 1 DN
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(1)
+ .build();
+ cluster.waitForClusterToBeReady();
+
+ // Stop the SCM
+ StorageContainerManager scm = cluster.getStorageContainerManager();
+ scm.stop();
+
+ // Restart DN
+ cluster.restartHddsDatanode(0, false);
+
+ // DN should be in GETVERSION state till the SCM is restarted.
+ // Check DN endpoint state for 20 seconds
+ DatanodeStateMachine dnStateMachine = cluster.getHddsDatanodes().get(0)
+ .getDatanodeStateMachine();
+ for (int i = 0; i < 20; i++) {
+ for (EndpointStateMachine endpoint :
+ dnStateMachine.getConnectionManager().getValues()) {
+ Assert.assertEquals(
+ EndpointStateMachine.EndPointStates.GETVERSION,
+ endpoint.getState());
+ }
+ Thread.sleep(1000);
+ }
+
+ // DN should successfully register with the SCM after SCM is restarted.
+ // Restart the SCM
+ cluster.restartStorageContainerManager();
+ // Wait for DN to register
+ cluster.waitForClusterToBeReady();
+ // DN should be in HEARTBEAT state after registering with the SCM
+ for (EndpointStateMachine endpoint :
+ dnStateMachine.getConnectionManager().getValues()) {
+ Assert.assertEquals(EndpointStateMachine.EndPointStates.HEARTBEAT,
+ endpoint.getState());
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDDS-786. Fix the findbugs for
SCMClientProtocolServer#getContainerWithPipeline.
Posted by ae...@apache.org.
HDDS-786. Fix the findbugs for SCMClientProtocolServer#getContainerWithPipeline.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e8ac14d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e8ac14d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e8ac14d
Branch: refs/heads/HDDS-4
Commit: 2e8ac14dcb57a0fe07b2119c26535c3541665b70
Parents: b13c567
Author: Yiqun Lin <yq...@apache.org>
Authored: Thu Nov 1 14:10:17 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Thu Nov 1 14:10:17 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e8ac14d/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index e92200a..58cb871 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -189,7 +189,6 @@ public class SCMClientProtocolServer implements
}
}
}
- String remoteUser = getRpcRemoteUsername();
getScm().checkAdminAccess(null);
return scm.getContainerManager()
.getContainerWithPipeline(ContainerID.valueof(containerID));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-13942. [JDK10] Fix javadoc
errors in hadoop-hdfs module. Contributed by Dinesh Chitlangia.
Posted by ae...@apache.org.
HDFS-13942. [JDK10] Fix javadoc errors in hadoop-hdfs module. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fac9f91b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fac9f91b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fac9f91b
Branch: refs/heads/HDDS-4
Commit: fac9f91b2944cee641049fffcafa6b65e0cf68f2
Parents: e4f22b0
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 31 14:43:58 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 31 14:43:58 2018 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 12 ++--
.../hadoop/hdfs/protocol/BlockListAsLongs.java | 2 +-
.../QJournalProtocolServerSideTranslatorPB.java | 2 +-
.../token/block/BlockTokenSecretManager.java | 2 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 15 ++---
.../server/blockmanagement/BlockManager.java | 26 +++++----
.../blockmanagement/BlockPlacementPolicy.java | 1 -
.../CombinedHostFileManager.java | 6 +-
.../blockmanagement/CorruptReplicasMap.java | 2 +-
.../blockmanagement/DatanodeAdminManager.java | 8 +--
.../server/blockmanagement/HostFileManager.java | 7 +--
.../hdfs/server/blockmanagement/HostSet.java | 8 +--
.../server/blockmanagement/SlowPeerTracker.java | 5 +-
.../server/datanode/BlockPoolSliceStorage.java | 60 ++++++++++++--------
.../server/datanode/BlockRecoveryWorker.java | 15 +++--
.../hdfs/server/datanode/BlockScanner.java | 6 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 10 ++--
.../hdfs/server/datanode/DataStorage.java | 4 +-
.../hdfs/server/datanode/DirectoryScanner.java | 1 -
.../hdfs/server/datanode/FileIoProvider.java | 3 -
.../hdfs/server/datanode/VolumeScanner.java | 4 +-
.../server/datanode/checker/AbstractFuture.java | 13 ++---
.../server/datanode/fsdataset/FsDatasetSpi.java | 12 ++--
.../server/datanode/fsdataset/FsVolumeSpi.java | 13 +++--
.../datanode/metrics/OutlierDetector.java | 3 +-
.../diskbalancer/DiskBalancerException.java | 1 -
.../datamodel/DiskBalancerCluster.java | 11 ++--
.../datamodel/DiskBalancerDataNode.java | 10 ++--
.../diskbalancer/planner/GreedyPlanner.java | 2 +-
.../hadoop/hdfs/server/namenode/AclStorage.java | 18 +++---
.../server/namenode/EncryptionZoneManager.java | 42 +++++++++-----
.../hdfs/server/namenode/FSDirectory.java | 8 +--
.../hdfs/server/namenode/FSNamesystem.java | 24 ++++++--
.../hadoop/hdfs/server/namenode/INode.java | 4 +-
.../hdfs/server/namenode/INodeReference.java | 6 +-
.../hdfs/server/namenode/INodesInPath.java | 4 +-
.../hdfs/server/namenode/JournalManager.java | 2 +-
.../hdfs/server/namenode/LeaseManager.java | 2 +-
.../server/namenode/MetaRecoveryContext.java | 2 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 6 +-
.../hdfs/server/namenode/NamenodeFsck.java | 9 ++-
.../hadoop/hdfs/server/namenode/Quota.java | 5 +-
.../server/namenode/ReencryptionHandler.java | 2 +-
.../server/namenode/XAttrPermissionFilter.java | 4 +-
.../hdfs/server/namenode/XAttrStorage.java | 8 +--
.../snapshot/AbstractINodeDiffList.java | 8 +--
.../namenode/snapshot/DiffListBySkipList.java | 9 +--
.../sps/BlockStorageMovementNeeded.java | 5 +-
.../namenode/sps/DatanodeCacheManager.java | 2 +-
.../sps/StoragePolicySatisfyManager.java | 14 +++--
.../startupprogress/StartupProgressView.java | 4 +-
.../server/namenode/top/metrics/TopMetrics.java | 17 ++++--
.../namenode/top/window/RollingWindow.java | 18 +++---
.../top/window/RollingWindowManager.java | 2 +-
.../protocol/BlockStorageMovementCommand.java | 11 ++--
.../hdfs/server/protocol/DatanodeProtocol.java | 2 +-
.../hdfs/server/protocol/NamenodeProtocol.java | 5 +-
.../sps/ExternalSPSBlockMoveTaskHandler.java | 2 +
.../org/apache/hadoop/hdfs/tools/DFSck.java | 13 +++--
.../offlineEditsViewer/OfflineEditsViewer.java | 4 +-
.../offlineEditsViewer/OfflineEditsVisitor.java | 2 +-
.../StatisticsEditsVisitor.java | 4 +-
.../NameDistributionVisitor.java | 4 +-
.../java/org/apache/hadoop/hdfs/util/Diff.java | 16 +++---
.../org/apache/hadoop/hdfs/util/XMLUtils.java | 4 +-
65 files changed, 310 insertions(+), 246 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 6dd366f..8627268 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -160,7 +160,8 @@ public class DFSUtil {
/**
* Comparator for sorting DataNodeInfo[] based on
* stale, decommissioned and entering_maintenance states.
- * Order: live -> stale -> entering_maintenance -> decommissioned
+ * Order: live {@literal ->} stale {@literal ->} entering_maintenance
+ * {@literal ->} decommissioned
*/
@InterfaceAudience.Private
public static class ServiceAndStaleComparator extends ServiceComparator {
@@ -390,7 +391,8 @@ public class DFSUtil {
* @param conf Configuration
* @param nsId the nameservice whose NNs addresses we want.
* @param defaultValue default address to return in case key is not found.
- * @return A map from nnId -> RPC address of each NN in the nameservice.
+ * @return A map from nnId {@literal ->} RPC address of each NN in the
+ * nameservice.
*/
public static Map<String, InetSocketAddress> getRpcAddressesForNameserviceId(
Configuration conf, String nsId, String defaultValue) {
@@ -1289,7 +1291,8 @@ public class DFSUtil {
* @param conf configuration
* @param protocol Protocol interface
* @param service service that implements the protocol
- * @param server RPC server to which the protocol & implementation is added to
+ * @param server RPC server to which the protocol & implementation is
+ * added to
* @throws IOException
*/
public static void addPBProtocol(Configuration conf, Class<?> protocol,
@@ -1357,7 +1360,8 @@ public class DFSUtil {
* @param conf Configuration
* @param nsId the nameservice whose NNs addresses we want.
* @param defaultValue default address to return in case key is not found.
- * @return A map from nnId -> Web address of each NN in the nameservice.
+ * @return A map from nnId {@literal ->} Web address of each NN in the
+ * nameservice.
*/
public static Map<String, InetSocketAddress> getWebAddressesForNameserviceId(
Configuration conf, String nsId, String defaultValue) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
index 8f482e3..77e40b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
@@ -201,7 +201,7 @@ public abstract class BlockListAsLongs implements Iterable<BlockReportReplica> {
/**
* Very efficient encoding of the block report into a ByteString to avoid
* the overhead of protobuf repeating fields. Primitive repeating fields
- * require re-allocs of an ArrayList<Long> and the associated (un)boxing
+ * require re-allocs of an ArrayList<Long> and the associated (un)boxing
* overhead which puts pressure on GC.
*
* The structure of the buffer is as follows:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
index 2ad19da..61e8fa3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
@@ -168,7 +168,7 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
return VOID_JOURNAL_RESPONSE;
}
- /** @see JournalProtocol#heartbeat */
+ /** @see QJournalProtocol#heartbeat */
@Override
public HeartbeatResponseProto heartbeat(RpcController controller,
HeartbeatRequestProto req) throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
index 3b2e8d2..52bc52d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
@@ -220,7 +220,7 @@ public class BlockTokenSecretManager extends
}
/**
- * Update block keys if update time > update interval.
+ * Update block keys if update time {@literal >} update interval.
* @return true if the keys are updated.
*/
public synchronized boolean updateKeys(final long updateTime) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index a58e391..d21d13c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -78,7 +78,7 @@ import com.google.common.base.Preconditions;
* <p>SYNOPSIS
* <pre>
* To start:
- * bin/start-balancer.sh [-threshold <threshold>]
+ * bin/start-balancer.sh [-threshold {@literal <threshold>}]
* Example: bin/ start-balancer.sh
* start the balancer with a default threshold of 10%
* bin/ start-balancer.sh -threshold 5
@@ -113,13 +113,14 @@ import com.google.common.base.Preconditions;
* <p>A system property that limits the balancer's use of bandwidth is
* defined in the default configuration file:
* <pre>
- * <property>
- * <name>dfs.datanode.balance.bandwidthPerSec</name>
- * <value>1048576</value>
- * <description> Specifies the maximum bandwidth that each datanode
+ * <property>
+ * <name>dfs.datanode.balance.bandwidthPerSec</name>
+ * <value>1048576</value>
+ * <description> Specifies the maximum bandwidth that each datanode
* can utilize for the balancing purpose in term of the number of bytes
- * per second. </description>
- * </property>
+ * per second.
+ * </description>
+ * </property>
* </pre>
*
* <p>This property determines the maximum speed at which a block will be
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5e14247..d74b523 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -143,12 +143,13 @@ import org.slf4j.LoggerFactory;
* If any of the replica is in maintenance mode, the safety property
* is extended as follows. These property still apply for the case of zero
* maintenance replicas, thus we can use these safe property for all scenarios.
- * a. # of live replicas >= # of min replication for maintenance.
- * b. # of live replicas <= # of expected redundancy.
- * c. # of live replicas and maintenance replicas >= # of expected redundancy.
+ * a. # of live replicas >= # of min replication for maintenance.
+ * b. # of live replicas <= # of expected redundancy.
+ * c. # of live replicas and maintenance replicas >= # of expected
+ * redundancy.
*
* For regular replication, # of min live replicas for maintenance is determined
- * by DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY. This number has to <=
+ * by DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY. This number has to <=
* DFS_NAMENODE_REPLICATION_MIN_KEY.
* For erasure encoding, # of min live replicas for maintenance is
* BlockInfoStriped#getRealDataBlockNum.
@@ -305,7 +306,7 @@ public class BlockManager implements BlockStatsMXBean {
private final double storageInfoDefragmentRatio;
/**
- * Mapping: Block -> { BlockCollection, datanodes, self ref }
+ * Mapping: Block {@literal ->} { BlockCollection, datanodes, self ref }
* Updated only in response to client-sent information.
*/
final BlocksMap blocksMap;
@@ -321,7 +322,9 @@ public class BlockManager implements BlockStatsMXBean {
private final BlockReportProcessingThread blockReportThread =
new BlockReportProcessingThread();
- /** Store blocks -> datanodedescriptor(s) map of corrupt replicas */
+ /**
+ * Store blocks {@literal ->} datanodedescriptor(s) map of corrupt replicas.
+ */
final CorruptReplicasMap corruptReplicas = new CorruptReplicasMap();
/**
@@ -2105,7 +2108,7 @@ public class BlockManager implements BlockStatsMXBean {
* Choose target datanodes for creating a new block.
*
* @throws IOException
- * if the number of targets < minimum replication.
+ * if the number of targets {@literal <} minimum replication.
* @see BlockPlacementPolicy#chooseTarget(String, int, Node,
* Set, long, List, BlockStoragePolicy, EnumSet)
*/
@@ -2487,7 +2490,8 @@ public class BlockManager implements BlockStatsMXBean {
/**
* The given storage is reporting all its blocks.
- * Update the (storage-->block list) and (block-->storage list) maps.
+ * Update the (storage{@literal -->}block list) and
+ * (block{@literal -->}storage list) maps.
*
* @return true if all known storages of the given DN have finished reporting.
* @throws IOException
@@ -3777,8 +3781,8 @@ public class BlockManager implements BlockStatsMXBean {
}
/**
- * Modify (block-->datanode) map. Possibly generate replication tasks, if the
- * removed block is still valid.
+ * Modify (block{@literal -->}datanode) map. Possibly generate replication
+ * tasks, if the removed block is still valid.
*/
public void removeStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) {
blockLog.debug("BLOCK* removeStoredBlock: {} from {}", storedBlock, node);
@@ -4341,7 +4345,7 @@ public class BlockManager implements BlockStatsMXBean {
}
/**
- * Get blocks to invalidate for <i>nodeId</i>
+ * Get blocks to invalidate for {@code nodeId}
* in {@link #invalidateBlocks}.
*
* @return number of blocks scheduled for removal during this iteration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
index 23e3e40..897bf69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
@@ -152,7 +152,6 @@ public abstract class BlockPlacementPolicy {
/**
* Check if the move is allowed. Used by balancer and other tools.
- * @
*
* @param candidates all replicas including source and target
* @param source source replica of the move
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java
index d6a0972..d607789 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java
@@ -47,13 +47,11 @@ import org.apache.hadoop.hdfs.util.CombinedHostsFileReader;
/**
* This class manages datanode configuration using a json file.
* Please refer to {@link CombinedHostsFileReader} for the json format.
- * <p/>
- * <p/>
+ * <p>
* Entries may or may not specify a port. If they don't, we consider
* them to apply to every DataNode on that host. The code canonicalizes the
* entries into IP addresses.
- * <p/>
- * <p/>
+ * <p>
* The code ignores all entries that the DNS fails to resolve their IP
* addresses. This is okay because by default the NN rejects the registrations
* of DNs when it fails to do a forward and reverse lookup. Note that DNS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
index fe1224c..fc31584 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
@@ -38,7 +38,7 @@ import com.google.common.annotations.VisibleForTesting;
* corrupt. While reporting replicas of a Block, we hide any corrupt
* copies. These copies are removed once Block is found to have
* expected number of good replicas.
- * Mapping: Block -> TreeSet<DatanodeDescriptor>
+ * Mapping: Block {@literal -> TreeSet<DatanodeDescriptor>}
*/
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java
index a1dff08..abc0f7c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java
@@ -57,7 +57,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
* Manages decommissioning and maintenance state for DataNodes. A background
* monitor thread periodically checks the status of DataNodes that are
* decommissioning or entering maintenance state.
- * <p/>
+ * <p>
* A DataNode can be decommissioned in a few situations:
* <ul>
* <li>If a DN is dead, it is decommissioned immediately.</li>
@@ -72,11 +72,11 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
* determine if they can be DECOMMISSIONED. The monitor also prunes this list
* as blocks become replicated, so monitor scans will become more efficient
* over time.
- * <p/>
+ * <p>
* DECOMMISSION_INPROGRESS nodes that become dead do not progress to
* DECOMMISSIONED until they become live again. This prevents potential
* durability loss for singly-replicated blocks (see HDFS-6791).
- * <p/>
+ * <p>
* DataNodes can also be put under maintenance state for any short duration
* maintenance operations. Unlike decommissioning, blocks are not always
* re-replicated for the DataNodes to enter maintenance state. When the
@@ -88,7 +88,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
* of maintenance expiry time. When DataNodes don't transition or join the
* cluster back by expiry time, blocks are re-replicated just as in
* decommissioning case as to avoid read or write performance degradation.
- * <p/>
+ * <p>
* This class depends on the FSNamesystem lock for synchronization.
*/
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java
index b7bf674..4ead0ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java
@@ -33,17 +33,16 @@ import java.util.HashSet;
/**
* This class manages the include and exclude files for HDFS.
- * <p/>
+ * <p>
* These files control which DataNodes the NameNode expects to see in the
* cluster. Loosely speaking, the include file, if it exists and is not
* empty, is a list of everything we expect to see. The exclude file is
* a list of everything we want to ignore if we do see it.
- * <p/>
+ * <p>
* Entries may or may not specify a port. If they don't, we consider
* them to apply to every DataNode on that host. The code canonicalizes the
* entries into IP addresses.
- * <p/>
- * <p/>
+ * <p>
* The code ignores all entries that the DNS fails to resolve their IP
* addresses. This is okay because by default the NN rejects the registrations
* of DNs when it fails to do a forward and reverse lookup. Note that DNS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
index 958557b..cf7cfac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
@@ -35,9 +35,9 @@ import java.util.Map;
/**
* The HostSet allows efficient queries on matching wildcard addresses.
- * <p/>
+ * <p>
* For InetSocketAddress A and B with the same host address,
- * we define a partial order between A and B, A <= B iff A.getPort() == B
+ * we define a partial order between A and B, A <= B iff A.getPort() == B
* .getPort() || B.getPort() == 0.
*/
public class HostSet implements Iterable<InetSocketAddress> {
@@ -46,7 +46,7 @@ public class HostSet implements Iterable<InetSocketAddress> {
/**
* The function that checks whether there exists an entry foo in the set
- * so that foo <= addr.
+ * so that foo <= addr.
*/
boolean matchedBy(InetSocketAddress addr) {
Collection<Integer> ports = addrs.get(addr.getAddress());
@@ -56,7 +56,7 @@ public class HostSet implements Iterable<InetSocketAddress> {
/**
* The function that checks whether there exists an entry foo in the set
- * so that addr <= foo.
+ * so that addr <= foo.
*/
boolean match(InetSocketAddress addr) {
int port = addr.getPort();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
index 22983ea..03a6918 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
import org.apache.hadoop.util.Timer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,7 +83,7 @@ public class SlowPeerTracker {
/**
* Information about peers that have reported a node as being slow.
- * Each outer map entry is a map of (DatanodeId) -> (timestamp),
+ * Each outer map entry is a map of (DatanodeId) {@literal ->} (timestamp),
* mapping reporting nodes to the timestamp of the last report from
* that node.
*
@@ -146,7 +147,7 @@ public class SlowPeerTracker {
/**
* Retrieve all reports for all nodes. Stale reports are excluded.
*
- * @return map from SlowNodeId -> (set of nodes reporting peers).
+ * @return map from SlowNodeId {@literal ->} (set of nodes reporting peers).
*/
public Map<String, SortedSet<String>> getReportsForAllDataNodes() {
if (allReports.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
index 4fc47d8..539baf1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
@@ -56,9 +56,9 @@ import com.google.common.collect.Lists;
* block pool id, on this DataNode.
*
* This class supports the following functionality:
- * <ol>
+ * <ul>
* <li> Formatting a new block pool storage</li>
- * <li> Recovering a storage state to a consistent state (if possible></li>
+ * <li> Recovering a storage state to a consistent state (if possible)</li>
* <li> Taking a snapshot of the block pool during upgrade</li>
* <li> Rolling back a block pool to a previous snapshot</li>
* <li> Finalizing block storage by deletion of a snapshot</li>
@@ -139,11 +139,12 @@ public class BlockPoolSliceStorage extends Storage {
/**
* Load one storage directory. Recover from previous transitions if required.
- *
- * @param nsInfo namespace information
- * @param dataDir the root path of the storage directory
- * @param startOpt startup option
- * @return the StorageDirectory successfully loaded.
+ * @param nsInfo namespace information
+ * @param location the root path of the storage directory
+ * @param startOpt startup option
+ * @param callables list of callable storage directory
+ * @param conf configuration
+ * @return
* @throws IOException
*/
private StorageDirectory loadStorageDirectory(NamespaceInfo nsInfo,
@@ -205,8 +206,10 @@ public class BlockPoolSliceStorage extends Storage {
* data volume.
*
* @param nsInfo namespace information
- * @param dataDirs storage directories of block pool
+ * @param location storage directories of block pool
* @param startOpt startup option
+ * @param callables list of callable storage directory
+ * @param conf configuration
* @return an array of loaded block pool directories.
* @throws IOException on error
*/
@@ -240,8 +243,10 @@ public class BlockPoolSliceStorage extends Storage {
* data volume.
*
* @param nsInfo namespace information
- * @param dataDirs storage directories of block pool
+ * @param location storage directories of block pool
* @param startOpt startup option
+ * @param callables list of callable storage directory
+ * @param conf configuration
* @throws IOException on error
*/
List<StorageDirectory> recoverTransitionRead(NamespaceInfo nsInfo,
@@ -348,13 +353,18 @@ public class BlockPoolSliceStorage extends Storage {
* Analyze whether a transition of the BP state is required and
* perform it if necessary.
* <br>
- * Rollback if previousLV >= LAYOUT_VERSION && prevCTime <= namenode.cTime.
- * Upgrade if this.LV > LAYOUT_VERSION || this.cTime < namenode.cTime Regular
- * startup if this.LV = LAYOUT_VERSION && this.cTime = namenode.cTime
+ * Rollback if:
+ * previousLV >= LAYOUT_VERSION && prevCTime <= namenode.cTime.
+ * Upgrade if:
+ * this.LV > LAYOUT_VERSION || this.cTime < namenode.cTime
+ * Regular startup if:
+ * this.LV = LAYOUT_VERSION && this.cTime = namenode.cTime
*
- * @param sd storage directory <SD>/current/<bpid>
+ * @param sd storage directory @{literal <SD>/current/<bpid>}
* @param nsInfo namespace info
* @param startOpt startup option
+ * @param callables list of callable storage directory
+ * @param conf configuration
* @return true if the new properties has been written.
*/
private boolean doTransition(StorageDirectory sd, NamespaceInfo nsInfo,
@@ -416,20 +426,20 @@ public class BlockPoolSliceStorage extends Storage {
}
/**
- * Upgrade to any release after 0.22 (0.22 included) release e.g. 0.22 => 0.23
+ * Upgrade to any release after 0.22 (0.22 included) release
+ * e.g. 0.22 => 0.23
* Upgrade procedure is as follows:
* <ol>
- * <li>If <SD>/current/<bpid>/previous exists then delete it</li>
- * <li>Rename <SD>/current/<bpid>/current to
- * <SD>/current/bpid/current/previous.tmp</li>
- * <li>Create new <SD>current/<bpid>/current directory</li>
- * <ol>
+ * <li>If {@literal <SD>/current/<bpid>/previous} exists then delete it</li>
+ * <li>Rename {@literal <SD>/current/<bpid>/current} to
+ * {@literal <SD>/current/bpid/current/previous.tmp}</li>
+ * <li>Create new {@literal <SD>current/<bpid>/current} directory</li>
* <li>Hard links for block files are created from previous.tmp to current</li>
* <li>Save new version file in current directory</li>
+ * <li>Rename previous.tmp to previous</li>
* </ol>
- * <li>Rename previous.tmp to previous</li> </ol>
*
- * @param bpSd storage directory <SD>/current/<bpid>
+ * @param bpSd storage directory {@literal <SD>/current/<bpid>}
* @param nsInfo Namespace Info from the namenode
* @throws IOException on error
*/
@@ -777,12 +787,12 @@ public class BlockPoolSliceStorage extends Storage {
}
/**
- * Get a target subdirectory under current/ for a given block file that is being
- * restored from trash.
+ * Get a target subdirectory under current/ for a given block file that is
+ * being restored from trash.
*
* The subdirectory structure under trash/ mirrors that under current/ to keep
* implicit memory of where the files are to be restored.
- *
+ * @param blockFile block file that is being restored from trash.
* @return the target directory to restore a previously deleted block file.
*/
@VisibleForTesting
@@ -847,6 +857,7 @@ public class BlockPoolSliceStorage extends Storage {
/**
* Create a rolling upgrade marker file for each BP storage root, if it
* does not exist already.
+ * @param dnStorageDirs
*/
public void setRollingUpgradeMarkers(List<StorageDirectory> dnStorageDirs)
throws IOException {
@@ -872,6 +883,7 @@ public class BlockPoolSliceStorage extends Storage {
* Check whether the rolling upgrade marker file exists for each BP storage
* root. If it does exist, then the marker file is cleared and more
* importantly the layout upgrade is finalized.
+ * @param dnStorageDirs
*/
public void clearRollingUpgradeMarkers(List<StorageDirectory> dnStorageDirs)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
index 34f6c33..fe0c7f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
@@ -338,19 +338,24 @@ public class BlockRecoveryWorker {
/**
* blk_0 blk_1 blk_2 blk_3 blk_4 blk_5 blk_6 blk_7 blk_8
- * 64k 64k 64k 64k 64k 64k 64k 64k 64k <-- stripe_0
+ * 64k 64k 64k 64k 64k 64k 64k 64k 64k <--
+ * stripe_0
* 64k 64k 64k 64k 64k 64k 64k 64k 64k
- * 64k 64k 64k 64k 64k 64k 64k 61k <-- startStripeIdx
+ * 64k 64k 64k 64k 64k 64k 64k 61k <--
+ * startStripeIdx
* 64k 64k 64k 64k 64k 64k 64k
* 64k 64k 64k 64k 64k 64k 59k
* 64k 64k 64k 64k 64k 64k
- * 64k 64k 64k 64k 64k 64k <-- last full stripe
- * 64k 64k 13k 64k 55k 3k <-- target last stripe
+ * 64k 64k 64k 64k 64k 64k <--
+ * last full stripe
+ * 64k 64k 13k 64k 55k 3k <--
+ * target last stripe
* 64k 64k 64k 1k
* 64k 64k 58k
* 64k 64k
* 64k 19k
- * 64k <-- total visible stripe
+ * 64k <--
+ * total visible stripe
*
* Due to different speed of streamers, the internal blocks in a block group
* could have different lengths when the block group isn't ended normally.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
index 8081895..6b1b96f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
@@ -189,7 +189,7 @@ public class BlockScanner {
}
/**
- * Returns true if the block scanner is enabled.<p/>
+ * Returns true if the block scanner is enabled.
*
* If the block scanner is disabled, no volume scanners will be created, and
* no threads will start.
@@ -234,7 +234,7 @@ public class BlockScanner {
}
/**
- * Stops and removes a volume scanner.<p/>
+ * Stops and removes a volume scanner.
*
* This function will block until the volume scanner has stopped.
*
@@ -260,7 +260,7 @@ public class BlockScanner {
}
/**
- * Stops and removes all volume scanners.<p/>
+ * Stops and removes all volume scanners.
*
* This function will block until all the volume scanners have stopped.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 40f80a9..99c0a87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -242,7 +242,7 @@ import org.slf4j.LoggerFactory;
* DataNodes.
*
* The DataNode maintains just one critical table:
- * block-> stream of bytes (of BLOCK_SIZE or less)
+ * block{@literal ->} stream of bytes (of BLOCK_SIZE or less)
*
* This info is stored on a local disk. The DataNode
* reports the table's contents to the NameNode upon startup
@@ -527,7 +527,7 @@ public class DataNode extends ReconfigurableBase
}
/**
- * {@inheritdoc}.
+ * {@inheritDoc }.
*/
@Override
public String reconfigurePropertyImpl(String property, String newVal)
@@ -2713,7 +2713,8 @@ public class DataNode extends ReconfigurableBase
return locations;
}
- /** Instantiate & Start a single datanode daemon and wait for it to finish.
+ /** Instantiate & Start a single datanode daemon and wait for it to
+ * finish.
* If this thread is specifically interrupted, it will stop waiting.
*/
@VisibleForTesting
@@ -2722,7 +2723,8 @@ public class DataNode extends ReconfigurableBase
return createDataNode(args, conf, null);
}
- /** Instantiate & Start a single datanode daemon and wait for it to finish.
+ /** Instantiate & Start a single datanode daemon and wait for it to
+ * finish.
* If this thread is specifically interrupted, it will stop waiting.
*/
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index a85ae32..a803c0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -215,7 +215,9 @@ public class DataStorage extends Storage {
/**
* VolumeBuilder holds the metadata (e.g., the storage directories) of the
- * prepared volume returned from {@link prepareVolume()}. Calling {@link build()}
+ * prepared volume returned from
+ * {@link #prepareVolume(DataNode, StorageLocation, List)}.
+ * Calling {@link VolumeBuilder#build()}
* to add the metadata to {@link DataStorage} so that this prepared volume can
* be active.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 7ae9e45..445e021 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -280,7 +280,6 @@ public class DirectoryScanner implements Runnable {
/**
* Create a new directory scanner, but don't cycle it running yet.
*
- * @param datanode the parent datanode
* @param dataset the dataset to scan
* @param conf the Configuration object
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index b8e08d0..6349062 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -178,7 +178,6 @@ public class FileIoProvider {
* Call sync_file_range on the given file descriptor.
*
* @param volume target volume. null if unavailable.
- * @throws IOException
*/
public void syncFileRange(
@Nullable FsVolumeSpi volume, FileDescriptor outFd,
@@ -198,7 +197,6 @@ public class FileIoProvider {
* Call posix_fadvise on the given file descriptor.
*
* @param volume target volume. null if unavailable.
- * @throws IOException
*/
public void posixFadvise(
@Nullable FsVolumeSpi volume, String identifier, FileDescriptor outFd,
@@ -394,7 +392,6 @@ public class FileIoProvider {
* @param volume target volume. null if unavailable.
* @param fd File descriptor object.
* @return FileOutputStream to the given file object.
- * @throws FileNotFoundException
*/
public FileOutputStream getFileOutputStream(
@Nullable FsVolumeSpi volume, FileDescriptor fd) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index 181ef80..e0afb9b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -45,8 +45,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * VolumeScanner scans a single volume. Each VolumeScanner has its own thread.<p/>
- * They are all managed by the DataNode's BlockScanner.
+ * VolumeScanner scans a single volume. Each VolumeScanner has its own thread.
+ * <p>They are all managed by the DataNode's BlockScanner.
*/
public class VolumeScanner extends Thread {
public static final Logger LOG =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
index 2719f71..ec2b656 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
@@ -24,7 +24,10 @@ import com.google.common.annotations.Beta;
import com.google.common.annotations.GwtCompatible;
import com.google.common.base.Preconditions;
import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
import com.google.common.util.concurrent.Uninterruptibles;
import static java.util.concurrent.atomic.AtomicReferenceFieldUpdater
.newUpdater;
@@ -52,9 +55,7 @@ import java.util.logging.Logger;
* {@link ListeningExecutorService}, and deriving a {@code Future} from an
* existing one, typically using methods like {@link Futures#transform
* (ListenableFuture, com.google.common.base.Function) Futures.transform}
- * and {@link Futures#catching(ListenableFuture, Class,
- * com.google.common.base.Function, java.util.concurrent.Executor)
- * Futures.catching}.
+ * and its overloaded versions.
* <p>
* <p>This class implements all methods in {@code ListenableFuture}.
* Subclasses should provide a way to set the result of the computation
@@ -1265,12 +1266,6 @@ public abstract class AbstractFuture<V> implements ListenableFuture<V> {
* r.run();
* }
* }}</pre>
- * <p>
- * <p>This should be preferred to {@link #newDirectExecutorService()}
- * because implementing the {@link ExecutorService} subinterface
- * necessitates significant performance overhead.
- *
- * @since 18.0
*/
public static Executor directExecutor() {
return DirectExecutor.INSTANCE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index f4bf839..bd9ed7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -192,7 +192,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
FsVolumeReferences getFsVolumeReferences();
/**
- * Add a new volume to the FsDataset.<p/>
+ * Add a new volume to the FsDataset.
*
* If the FSDataset supports block scanning, this function registers
* the new volume with the block scanner.
@@ -226,7 +226,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
/** @return the volume that contains a replica of the block. */
V getVolume(ExtendedBlock b);
- /** @return a volume information map (name => info). */
+ /** @return a volume information map (name {@literal =>} info). */
Map<String, Object> getVolumeInfoMap();
/**
@@ -273,7 +273,8 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
/**
* Get reference to the replica meta info in the replicasMap.
- * To be called from methods that are synchronized on {@link FSDataset}
+ * To be called from methods that are synchronized on
+ * implementations of {@link FsDatasetSpi}
* @return replica from the replicas map
*/
@Deprecated
@@ -394,7 +395,7 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
* Finalizes the block previously opened for writing using writeToBlock.
* The block size is what is in the parameter b and it must match the amount
* of data written
- * @param block Block to be finalized
+ * @param b Block to be finalized
* @param fsyncDir whether to sync the directory changes to durable device.
* @throws IOException
* @throws ReplicaNotFoundException if the replica can not be found when the
@@ -488,14 +489,13 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
/**
* Determine if the specified block is cached.
* @param bpid Block pool id
- * @param blockIds - block id
+ * @param blockId - block id
* @return true if the block is cached
*/
boolean isCached(String bpid, long blockId);
/**
* Check if all the data directories are healthy
- * @return A set of unhealthy data directories.
* @param failedVolumes
*/
void handleVolumeFailures(Set<FsVolumeSpi> failedVolumes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index 7329ba3..be978d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@ -98,17 +98,17 @@ public interface FsVolumeSpi
/**
* BlockIterator will return ExtendedBlock entries from a block pool in
- * this volume. The entries will be returned in sorted order.<p/>
+ * this volume. The entries will be returned in sorted order.<p>
*
* BlockIterator objects themselves do not always have internal
* synchronization, so they can only safely be used by a single thread at a
- * time.<p/>
+ * time.<p>
*
* Closing the iterator does not save it. You must call save to save it.
*/
interface BlockIterator extends Closeable {
/**
- * Get the next block.<p/>
+ * Get the next block.<p>
*
* Note that this block may be removed in between the time we list it,
* and the time the caller tries to use it, or it may represent a stale
@@ -146,7 +146,7 @@ public interface FsVolumeSpi
void save() throws IOException;
/**
- * Set the maximum staleness of entries that we will return.<p/>
+ * Set the maximum staleness of entries that we will return.<p>
*
* A maximum staleness of 0 means we will never return stale entries; a
* larger value will allow us to reduce resource consumption in exchange
@@ -211,12 +211,12 @@ public interface FsVolumeSpi
* Because millions of these structures may be created, we try to save
* memory here. So instead of storing full paths, we store path suffixes.
* The block file, if it exists, will have a path like this:
- * <volume_base_path>/<block_path>
+ * {@literal <volume_base_path>/<block_path>}
* So we don't need to store the volume path, since we already know what the
* volume is.
*
* The metadata file, if it exists, will have a path like this:
- * <volume_base_path>/<block_path>_<genstamp>.meta
+ * {@literal <volume_base_path>/<block_path>_<genstamp>.meta}
* So if we have a block file, there isn't any need to store the block path
* again.
*
@@ -439,6 +439,7 @@ public interface FsVolumeSpi
* @param bpid block pool id to scan
* @param report the list onto which blocks reports are placed
* @param reportCompiler
+ * @throws InterruptedException
* @throws IOException
*/
void compileReport(String bpid,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
index 771a17b..401fc8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
@@ -94,7 +94,8 @@ public class OutlierDetector {
/**
* Return a set of nodes/ disks whose latency is much higher than
- * their counterparts. The input is a map of (resource -> aggregate latency)
+ * their counterparts. The input is a map of (resource {@literal ->} aggregate
+ * latency)
* entries.
*
* The aggregate may be an arithmetic mean or a percentile e.g.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
index 642cf21..7824f95 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
@@ -60,7 +60,6 @@ public class DiskBalancerException extends IOException {
/**
* Constructs an {@code IOException} with the specified detail message and
* cause.
- * <p/>
* <p> Note that the detail message associated with {@code cause} is
* <i>not</i>
* automatically incorporated into this exception's detail message.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
index 8de19aa..1307983 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
@@ -50,21 +50,20 @@ import java.util.concurrent.Future;
/**
* DiskBalancerCluster represents the nodes that we are working against.
- * <p/>
+ * <p>
* Please Note :
- * <p/>
* Semantics of inclusionList and exclusionLists.
- * <p/>
+ * <p>
* If a non-empty inclusionList is specified then the diskBalancer assumes that
* the user is only interested in processing that list of nodes. This node list
* is checked against the exclusionList and only the nodes in inclusionList but
* not in exclusionList is processed.
- * <p/>
+ * <p>
* if inclusionList is empty, then we assume that all live nodes in the nodes is
* to be processed by diskBalancer. In that case diskBalancer will avoid any
* nodes specified in the exclusionList but will process all nodes in the
* cluster.
- * <p/>
+ * <p>
* In other words, an empty inclusionList is means all the nodes otherwise
* only a given list is processed and ExclusionList is always honored.
*/
@@ -291,7 +290,7 @@ public class DiskBalancerCluster {
/**
* Compute plan takes a node and constructs a planner that creates a plan that
* we would like to follow.
- * <p/>
+ * <p>
* This function creates a thread pool and executes a planner on each node
* that we are supposed to plan for. Each of these planners return a NodePlan
* that we can persist or schedule for execution with a diskBalancer
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java
index a200f4d..6cf244b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java
@@ -186,7 +186,7 @@ public class DiskBalancerDataNode implements Comparable<DiskBalancerDataNode> {
}
/**
- * returns NodeDataDensity Metric.
+ * Returns NodeDataDensity Metric.
*
* @return float
*/
@@ -195,8 +195,8 @@ public class DiskBalancerDataNode implements Comparable<DiskBalancerDataNode> {
}
/**
- * computes nodes data density.
- * <p/>
+ * Computes nodes data density.
+ *
* This metric allows us to compare different nodes and how well the data is
* spread across a set of volumes inside the node.
*/
@@ -231,8 +231,8 @@ public class DiskBalancerDataNode implements Comparable<DiskBalancerDataNode> {
/**
* Adds a volume to the DataNode.
- * <p/>
- * it is assumed that we have one thread per node hence this call is not
+ *
+ * It is assumed that we have one thread per node hence this call is not
* synchronised neither is the map is protected.
*
* @param volume - volume
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
index fb83eeb..568c1e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
@@ -34,7 +34,7 @@ import java.util.TreeSet;
/**
* Greedy Planner is a simple planner that computes the largest possible move at
* any point of time given a volumeSet.
- * <p/>
+ * <p>
* This is done by choosing the disks with largest amount of data above and
* below the idealStorage and then a move is scheduled between them.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
index 10ecc23..fa268c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
@@ -156,7 +156,7 @@ public final class AclStorage {
*
* @param inode INode to read
* @param snapshotId int ID of snapshot to read
- * @return List<AclEntry> containing extended inode ACL entries
+ * @return {@literal List<AclEntry>} containing extended inode ACL entries
*/
public static List<AclEntry> readINodeAcl(INode inode, int snapshotId) {
AclFeature f = inode.getAclFeature(snapshotId);
@@ -167,7 +167,7 @@ public final class AclStorage {
* Reads the existing extended ACL entries of an INodeAttribute object.
*
* @param inodeAttr INode to read
- * @return List<AclEntry> containing extended inode ACL entries
+ * @return {@code List<AclEntry>} containing extended inode ACL entries
*/
public static List<AclEntry> readINodeAcl(INodeAttributes inodeAttr) {
AclFeature f = inodeAttr.getAclFeature();
@@ -175,7 +175,7 @@ public final class AclStorage {
}
/**
- * Build list of AclEntries from the AclFeature
+ * Build list of AclEntries from the {@link AclFeature}
* @param aclFeature AclFeature
* @return List of entries
*/
@@ -204,7 +204,7 @@ public final class AclStorage {
* ACL modification APIs, which always apply a delta on top of current state.
*
* @param inode INode to read
- * @return List<AclEntry> containing all logical inode ACL entries
+ * @return {@code List<AclEntry>} containing all logical inode ACL entries
*/
public static List<AclEntry> readINodeLogicalAcl(INode inode) {
FsPermission perm = inode.getFsPermission();
@@ -262,7 +262,7 @@ public final class AclStorage {
* {@link AclFeature}.
*
* @param inode INode to update
- * @param newAcl List<AclEntry> containing new ACL entries
+ * @param newAcl {@code List<AclEntry>} containing new ACL entries
* @param snapshotId int latest snapshot ID of inode
* @throws AclException if the ACL is invalid for the given inode
* @throws QuotaExceededException if quota limit is exceeded
@@ -312,8 +312,8 @@ public final class AclStorage {
/**
* Creates an AclFeature from the given ACL entries.
*
- * @param accessEntries List<AclEntry> access ACL entries
- * @param defaultEntries List<AclEntry> default ACL entries
+ * @param accessEntries {@code List<AclEntry>} access ACL entries
+ * @param defaultEntries {@code List<AclEntry>} default ACL entries
* @return AclFeature containing the required ACL entries
*/
private static AclFeature createAclFeature(List<AclEntry> accessEntries,
@@ -347,7 +347,7 @@ public final class AclStorage {
* POSIX ACLs model, which presents the mask as the permissions of the group
* class.
*
- * @param accessEntries List<AclEntry> access ACL entries
+ * @param accessEntries {@code List<AclEntry>} access ACL entries
* @param existingPerm FsPermission existing permissions
* @return FsPermission new permissions
*/
@@ -365,7 +365,7 @@ public final class AclStorage {
* group and other permissions are in order. Also preserve sticky bit and
* toggle ACL bit off.
*
- * @param accessEntries List<AclEntry> access ACL entries
+ * @param accessEntries {@code List<AclEntry>} access ACL entries
* @param existingPerm FsPermission existing permissions
* @return FsPermission new permissions
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 5604a21..8fa9578 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -67,7 +67,7 @@ import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants
/**
* Manages the list of encryption zones in the filesystem.
- * <p/>
+ * <p>
* The EncryptionZoneManager has its own lock, but relies on the FSDirectory
* lock being held for many operations. The FSDirectory lock should not be
* taken if the manager lock is already held.
@@ -294,7 +294,7 @@ public class EncryptionZoneManager {
/**
* Add a new encryption zone.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*
* @param inodeId of the encryption zone
@@ -308,7 +308,7 @@ public class EncryptionZoneManager {
/**
* Add a new encryption zone.
- * <p/>
+ * <p>
* Does not assume that the FSDirectory lock is held.
*
* @param inodeId of the encryption zone
@@ -326,7 +326,7 @@ public class EncryptionZoneManager {
/**
* Remove an encryption zone.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*/
void removeEncryptionZone(Long inodeId) {
@@ -344,7 +344,7 @@ public class EncryptionZoneManager {
/**
* Returns true if an IIP is within an encryption zone.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*/
boolean isInAnEZ(INodesInPath iip) throws UnresolvedLinkException,
@@ -355,7 +355,7 @@ public class EncryptionZoneManager {
/**
* Returns the full path from an INode id.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*/
String getFullPathName(Long nodeId) {
@@ -370,7 +370,7 @@ public class EncryptionZoneManager {
/**
* Get the key name for an encryption zone. Returns null if <tt>iip</tt> is
* not within an encryption zone.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*/
String getKeyName(final INodesInPath iip) throws IOException {
@@ -385,7 +385,7 @@ public class EncryptionZoneManager {
/**
* Looks up the EncryptionZoneInt for a path within an encryption zone.
* Returns null if path is not within an EZ.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*/
private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip)
@@ -434,7 +434,7 @@ public class EncryptionZoneManager {
* Looks up the nearest ancestor EncryptionZoneInt that contains the given
* path (excluding itself).
* Returns null if path is not within an EZ, or the path is the root dir '/'
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*/
private EncryptionZoneInt getParentEncryptionZoneForPath(INodesInPath iip)
@@ -467,7 +467,7 @@ public class EncryptionZoneManager {
/**
* Throws an exception if the provided path cannot be renamed into the
* destination because of differing parent encryption zones.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*
* @param srcIIP source IIP
@@ -529,7 +529,7 @@ public class EncryptionZoneManager {
/**
* Create a new encryption zone.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*/
XAttr createEncryptionZone(INodesInPath srcIIP, CipherSuite suite,
@@ -573,7 +573,7 @@ public class EncryptionZoneManager {
/**
* Cursor-based listing of encryption zones.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
*/
BatchedListEntries<EncryptionZone> listEncryptionZones(long prevId)
@@ -621,6 +621,8 @@ public class EncryptionZoneManager {
* @param zoneId
* @param zonePath
* @return true if path resolve to the id, false if not.
+ * @throws AccessControlException
+ * @throws ParentNotDirectoryException
* @throws UnresolvedLinkException
*/
private boolean pathResolvesToId(final long zoneId, final String zonePath)
@@ -645,6 +647,9 @@ public class EncryptionZoneManager {
/**
* Re-encrypts the given encryption zone path. If the given path is not the
* root of an encryption zone, an exception is thrown.
+ * @param zoneIIP
+ * @param keyVersionName
+ * @throws IOException
*/
List<XAttr> reencryptEncryptionZone(final INodesInPath zoneIIP,
final String keyVersionName) throws IOException {
@@ -673,7 +678,9 @@ public class EncryptionZoneManager {
/**
* Cancels the currently-running re-encryption of the given encryption zone.
* If the given path is not the root of an encryption zone,
- * * an exception is thrown.
+ * an exception is thrown.
+ * @param zoneIIP
+ * @throws IOException
*/
List<XAttr> cancelReencryptEncryptionZone(final INodesInPath zoneIIP)
throws IOException {
@@ -693,8 +700,10 @@ public class EncryptionZoneManager {
/**
* Cursor-based listing of zone re-encryption status.
- * <p/>
+ * <p>
* Called while holding the FSDirectory lock.
+ * @param prevId
+ * @throws IOException
*/
BatchedListEntries<ZoneReencryptionStatus> listReencryptionStatus(
final long prevId) throws IOException {
@@ -735,6 +744,10 @@ public class EncryptionZoneManager {
/**
* Return whether an INode is an encryption zone root.
+ * @param inode
+ * @param name
+ * @return true when INode is an encryption zone root else false
+ * @throws FileNotFoundException
*/
boolean isEncryptionZoneRoot(final INode inode, final String name)
throws FileNotFoundException {
@@ -756,6 +769,7 @@ public class EncryptionZoneManager {
* Return whether an INode is an encryption zone root.
*
* @param inode the zone inode
+ * @param name
* @throws IOException if the inode is not a directory,
* or is a directory but not the root of an EZ.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 0140912..712a327 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -634,12 +634,10 @@ public class FSDirectory implements Closeable {
* no permission checks.
* @param src The path to resolve.
* @param dirOp The {@link DirOp} that controls additional checks.
- * @param resolveLink If false, only ancestor symlinks will be checked. If
- * true, the last inode will also be checked.
* @return if the path indicates an inode, return path after replacing up to
- * <inodeid> with the corresponding path of the inode, else the path
- * in {@code src} as is. If the path refers to a path in the "raw"
- * directory, return the non-raw pathname.
+ * {@code <inodeid>} with the corresponding path of the inode, else
+ * the path in {@code src} as is. If the path refers to a path in
+ * the "raw" directory, return the non-raw pathname.
* @throws FileNotFoundException
* @throws AccessControlException
* @throws ParentNotDirectoryException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index eda1164..d1904fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -341,10 +341,11 @@ import org.slf4j.LoggerFactory;
*
* This class and its contents keep:
*
- * 1) Valid fsname --> blocklist (kept on disk, logged)
+ * 1) Valid fsname {@literal -->} blocklist (kept on disk, logged)
* 2) Set of all valid blocks (inverted #1)
- * 3) block --> machinelist (kept in memory, rebuilt dynamically from reports)
- * 4) machine --> blocklist (inverted #2)
+ * 3) block {@literal -->} machinelist (kept in memory, rebuilt dynamically
+ * from reports)
+ * 4) machine {@literal -->} blocklist (inverted #2)
* 5) LRU cache of updated-heartbeat machines
*/
@InterfaceAudience.Private
@@ -1732,11 +1733,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
- * return a list of blocks & their locations on <code>datanode</code> whose
- * total size is <code>size</code>
+ * return a list of blocks & their locations on {@code datanode} whose
+ * total size is {@code size}
*
* @param datanode on which blocks are located
* @param size total size of blocks
+ * @param minimumBlockSize
*/
public BlocksWithLocations getBlocks(DatanodeID datanode, long size, long
minimumBlockSize) throws IOException {
@@ -1753,6 +1755,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Dump all metadata into specified file
+ * @param filename
*/
void metaSave(String filename) throws IOException {
String operationName = "metaSave";
@@ -1884,6 +1887,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/////////////////////////////////////////////////////////
/**
* Set permissions for an existing file.
+ * @param src
+ * @param permission
* @throws IOException
*/
void setPermission(String src, FsPermission permission) throws IOException {
@@ -1908,6 +1913,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Set owner for an existing file.
+ * @param src
+ * @param group
+ * @param username
* @throws IOException
*/
void setOwner(String src, String username, String group)
@@ -2188,6 +2196,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* @param replication new replication
* @return true if successful;
* false if file does not exist or is a directory
+ * @throws IOException
*/
boolean setReplication(final String src, final short replication)
throws IOException {
@@ -2219,6 +2228,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*
* @param src file/directory path
* @param policyName storage policy name
+ * @throws IOException
*/
void setStoragePolicy(String src, String policyName) throws IOException {
final String operationName = "setStoragePolicy";
@@ -2245,6 +2255,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* Satisfy the storage policy for a file or a directory.
*
* @param src file/directory path
+ * @throws IOException
*/
void satisfyStoragePolicy(String src, boolean logRetryCache)
throws IOException {
@@ -2295,6 +2306,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* unset storage policy set for a given file or a directory.
*
* @param src file/directory path
+ * @throws IOException
*/
void unsetStoragePolicy(String src) throws IOException {
final String operationName = "unsetStoragePolicy";
@@ -2321,6 +2333,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* @param src
* file/directory path
* @return storage policy object
+ * @throws IOException
*/
BlockStoragePolicy getStoragePolicy(String src) throws IOException {
checkOperation(OperationCategory.READ);
@@ -2336,6 +2349,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* @return All the existing block storage policies
+ * @throws IOException
*/
BlockStoragePolicy[] getStoragePolicies() throws IOException {
checkOperation(OperationCategory.READ);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 2123f4e..03b1ca3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -523,8 +523,8 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* 2. For a {@link WithName} node, since the node must be in a snapshot, we
* only count the quota usage for those nodes that still existed at the
* creation time of the snapshot associated with the {@link WithName} node.
- * We do not count in the size of the diff list.
- * <pre>
+ * We do not count in the size of the diff list.
+ * </pre>
*
* @param bsps Block storage policy suite to calculate intended storage type usage
* @param blockStoragePolicyId block storage policy id of the current INode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
index e4e14f7..8655bb3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
@@ -669,10 +669,10 @@ public abstract class INodeReference extends INode {
/**
* {@inheritDoc}
- * <br/>
+ * <br>
* To destroy a DstReference node, we first remove its link with the
- * referred node. If the reference number of the referred node is <= 0, we
- * destroy the subtree of the referred node. Otherwise, we clean the
+ * referred node. If the reference number of the referred node is <= 0,
+ * we destroy the subtree of the referred node. Otherwise, we clean the
* referred node's subtree and delete everything created after the last
* rename operation, i.e., everything outside of the scope of the prior
* WithName nodes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index 04fb50e..f072220 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -329,8 +329,8 @@ public class INodesInPath {
}
/**
- * @return the i-th inode if i >= 0;
- * otherwise, i < 0, return the (length + i)-th inode.
+ * @return the i-th inode if i {@literal >=} 0;
+ * otherwise, i {@literal <} 0, return the (length + i)-th inode.
*/
public INode getINode(int i) {
return inodes[(i < 0) ? inodes.length + i : i];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
index d6d2094..7331676 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
@@ -112,7 +112,7 @@ public interface JournalManager extends Closeable, FormatConfirmable,
void doRollback() throws IOException;
/**
- * Discard the segments whose first txid is >= the given txid.
+ * Discard the segments whose first txid is {@literal >=} the given txid.
* @param startTxId The given txid should be right at the segment boundary,
* i.e., it should be the first txid of some segment, if segment corresponding
* to the txid exists.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 31fb2bb..75db8de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -208,7 +208,7 @@ public class LeaseManager {
* read or write lock.
*
* @param ancestorDir the ancestor {@link INodeDirectory}
- * @return Set<INodesInPath>
+ * @return {@code Set<INodesInPath>}
*/
public Set<INodesInPath> getINodeWithLeases(final INodeDirectory
ancestorDir) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/MetaRecoveryContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/MetaRecoveryContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/MetaRecoveryContext.java
index dad5779..ead56a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/MetaRecoveryContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/MetaRecoveryContext.java
@@ -52,7 +52,7 @@ public final class MetaRecoveryContext {
* Display a prompt to the user and get his or her choice.
*
* @param prompt The prompt to display
- * @param default First choice (will be taken if autoChooseDefault is
+ * @param firstChoice First choice (will be taken if autoChooseDefault is
* true)
* @param choices Other choies
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 9f82cbd..b91e7ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -176,8 +176,8 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DE
* is a second backup/failover NameNode, or when using federated NameNodes.)
*
* The NameNode controls two critical tables:
- * 1) filename->blocksequence (namespace)
- * 2) block->machinelist ("inodes")
+ * 1) filename{@literal ->}blocksequence (namespace)
+ * 2) block{@literal ->}machinelist ("inodes")
*
* The first table is stored on disk and is very precious.
* The second table is rebuilt every time the NameNode comes up.
@@ -1111,7 +1111,7 @@ public class NameNode extends ReconfigurableBase implements
}
/**
- * @return NameNodeHttpServer, used by unit tests to ensure a full shutdown,
+ * NameNodeHttpServer, used by unit tests to ensure a full shutdown,
* so that no bind exception is thrown during restart.
*/
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 5d664cb..56607f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -94,14 +94,13 @@ import com.google.common.annotations.VisibleForTesting;
* <p>The tool scans all files and directories, starting from an indicated
* root path. The following abnormal conditions are detected and handled:</p>
* <ul>
- * <li>files with blocks that are completely missing from all datanodes.<br/>
+ * <li>files with blocks that are completely missing from all datanodes.<br>
* In this case the tool can perform one of the following actions:
* <ul>
- * <li>none ({@link #FIXING_NONE})</li>
* <li>move corrupted files to /lost+found directory on DFS
- * ({@link #FIXING_MOVE}). Remaining data blocks are saved as a
+ * ({@link #doMove}). Remaining data blocks are saved as a
* block chains, representing longest consecutive series of valid blocks.</li>
- * <li>delete corrupted files ({@link #FIXING_DELETE})</li>
+ * <li>delete corrupted files ({@link #doDelete})</li>
* </ul>
* </li>
* <li>detect files with under-replicated or over-replicated blocks</li>
@@ -201,7 +200,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
*/
NamenodeFsck(Configuration conf, NameNode namenode,
NetworkTopology networktopology,
- Map<String,String[]> pmap, PrintWriter out,
+ Map<String, String[]> pmap, PrintWriter out,
int totalDatanodes, InetAddress remoteAddress) {
this.conf = conf;
this.namenode = namenode;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HDDS-759. Create config settings
for SCM and OM DB directories. Contributed by Arpit Agarwal.
Posted by ae...@apache.org.
HDDS-759. Create config settings for SCM and OM DB directories. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08bb0362
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08bb0362
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08bb0362
Branch: refs/heads/HDDS-4
Commit: 08bb0362e0c57f562e2f2e366cba725649d1d9c8
Parents: 478b2cb
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Oct 31 11:23:15 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Oct 31 11:23:15 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 7 ++
.../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 ++
.../apache/hadoop/ozone/OzoneConfigKeys.java | 3 -
.../common/src/main/resources/ozone-default.xml | 45 +++++++---
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 16 ++--
.../ozone/container/common/SCMTestUtils.java | 3 +-
.../common/TestDatanodeStateMachine.java | 3 +-
.../container/ozoneimpl/TestOzoneContainer.java | 4 +-
.../apache/hadoop/hdds/server/ServerUtils.java | 49 +++++++++--
.../hdds/scm/block/DeletedBlockLogImpl.java | 10 +--
.../hdds/scm/container/SCMContainerManager.java | 4 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 6 +-
.../hadoop/hdds/scm/server/SCMStorage.java | 4 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 50 +++++++++++
.../hadoop/hdds/scm/block/TestBlockManager.java | 4 +-
.../hdds/scm/block/TestDeletedBlockLog.java | 4 +-
.../TestCloseContainerEventHandler.java | 4 +-
.../container/TestContainerReportHandler.java | 4 +-
.../scm/container/TestSCMContainerManager.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 4 +-
.../hdds/scm/node/TestDeadNodeHandler.java | 4 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 4 +-
.../ozone/container/common/TestEndPoint.java | 3 +-
.../java/org/apache/hadoop/ozone/OmUtils.java | 42 +++++++++
.../apache/hadoop/ozone/om/OMConfigKeys.java | 3 +
.../org/apache/hadoop/ozone/TestOmUtils.java | 91 ++++++++++++++++++++
.../scm/pipeline/TestSCMPipelineManager.java | 4 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 5 +-
.../hadoop/ozone/TestMiniOzoneCluster.java | 7 +-
.../ozone/TestStorageContainerManager.java | 9 +-
.../hadoop/ozone/om/TestOzoneManager.java | 3 +-
.../org/apache/hadoop/ozone/om/OMStorage.java | 5 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 5 +-
.../apache/hadoop/ozone/om/TestOmSQLCli.java | 3 +-
.../hadoop/ozone/scm/TestContainerSQLCli.java | 3 +-
36 files changed, 343 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 210b075..abacafe 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -97,4 +97,11 @@ public final class HddsConfigKeys {
"hdds.lock.max.concurrency";
public static final int HDDS_LOCK_MAX_CONCURRENCY_DEFAULT = 100;
+ // This configuration setting is used as a fallback location by all
+ // Ozone/HDDS services for their metadata. It is useful as a single
+ // config point for test/PoC clusters.
+ //
+ // In any real cluster where performance matters, the SCM, OM and DN
+ // metadata locations must be configured explicitly.
+ public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index 09fc75b..89edfdd 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -305,7 +305,7 @@ public final class HddsUtils {
public static String getDatanodeIdFilePath(Configuration conf) {
String dataNodeIDPath = conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID);
if (dataNodeIDPath == null) {
- String metaPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
+ String metaPath = conf.get(HddsConfigKeys.OZONE_METADATA_DIRS);
if (Strings.isNullOrEmpty(metaPath)) {
// this means meta data is not found, in theory should not happen at
// this point because should've failed earlier.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index f95b748..56692af 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -31,6 +31,11 @@ import java.util.concurrent.TimeUnit;
@InterfaceStability.Unstable
public final class ScmConfigKeys {
+ // Location of SCM DB files. For now we just support a single
+ // metadata dir but in future we may support multiple for redundancy or
+ // performance.
+ public static final String OZONE_SCM_DB_DIRS = "ozone.scm.db.dirs";
+
public static final String SCM_CONTAINER_CLIENT_STALE_THRESHOLD_KEY =
"scm.container.client.idle.threshold";
public static final String SCM_CONTAINER_CLIENT_STALE_THRESHOLD_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index c931dcf..3b4f017 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -74,9 +74,6 @@ public final class OzoneConfigKeys {
"ozone.trace.enabled";
public static final boolean OZONE_TRACE_ENABLED_DEFAULT = false;
- public static final String OZONE_METADATA_DIRS =
- "ozone.metadata.dirs";
-
public static final String OZONE_METADATA_STORE_IMPL =
"ozone.metastore.impl";
public static final String OZONE_METADATA_STORE_IMPL_LEVELDB =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 237f8d8..eb68662 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -491,21 +491,30 @@
</description>
</property>
<property>
+ <name>ozone.om.db.dirs</name>
+ <value/>
+ <tag>OZONE, OM, STORAGE, PERFORMANCE</tag>
+ <description>
+ Directory where the OzoneManager stores its metadata. This should
+ be specified as a single directory. If the directory does not
+ exist then the OM will attempt to create it.
+
+ If undefined, then the OM will log a warning and fallback to
+ ozone.metadata.dirs.
+ </description>
+ </property>
+ <property>
<name>ozone.metadata.dirs</name>
<value/>
- <tag>OZONE, OM, SCM, CONTAINER, REQUIRED, STORAGE</tag>
- <description>
- Ozone metadata is shared among OM, which acts as the namespace
- manager for ozone, SCM which acts as the block manager and data nodes
- which maintain the name of the key(Key Name and BlockIDs). This
- replicated and distributed metadata store is maintained under the
- directory pointed by this key. Since metadata can be I/O intensive, at
- least on OM and SCM we recommend having SSDs. If you have the luxury
- of mapping this path to SSDs on all machines in the cluster, that will
- be excellent.
+ <tag>OZONE, OM, SCM, CONTAINER, STORAGE</tag>
+ <description>
+ This setting is the fallback location for SCM, OM and DataNodes
+ to store their metadata. This setting may be used in test/PoC clusters
+ to simplify configuration.
- If Ratis metadata directories are not specified, Ratis server will emit a
- warning and use this path for storing its metadata too.
+ For production clusters or any time you care about performance, it is
+ recommended that ozone.om.db.dirs, ozone.scm.db.dirs and
+ dfs.container.ratis.datanode.storage.dir be configured separately.
</description>
</property>
<property>
@@ -533,7 +542,19 @@
Check the rocksdb documentation for more details.
</description>
</property>
+ <property>
+ <name>ozone.scm.db.dirs</name>
+ <value/>
+ <tag>OZONE, SCM, STORAGE, PERFORMANCE</tag>
+ <description>
+ Directory where the StorageContainerManager stores its metadata.
+ This should be specified as a single directory. If the directory
+ does not exist then the SCM will attempt to create it.
+ If undefined, then the SCM will log a warning and fallback to
+ ozone.metadata.dirs.
+ </description>
+ </property>
<property>
<name>ozone.scm.block.client.address</name>
<value/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index 395a77d..738a2ef 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -18,9 +18,10 @@
package org.apache.hadoop.hdds.scm;
import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.slf4j.Logger;
@@ -331,19 +332,16 @@ public final class HddsServerUtil {
}
public static String getOzoneDatanodeRatisDirectory(Configuration conf) {
- final String ratisDir = File.separator + "ratis";
String storageDir = conf.get(
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
if (Strings.isNullOrEmpty(storageDir)) {
- storageDir = conf.get(OzoneConfigKeys
- .OZONE_METADATA_DIRS);
- Preconditions.checkNotNull(storageDir, "ozone.metadata.dirs " +
- "cannot be null, Please check your configs.");
- storageDir = storageDir.concat(ratisDir);
LOG.warn("Storage directory for Ratis is not configured." +
- "Mapping Ratis storage under {}. It is a good idea " +
- "to map this to an SSD disk.", storageDir);
+ "Mapping Ratis storage under {}. It is a good idea " +
+ "to map this to an SSD disk. Falling back to {}",
+ storageDir, HddsConfigKeys.OZONE_METADATA_DIRS);
+ File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
+ storageDir = (new File (metaDirPath, "ratis")).getPath();
}
return storageDir;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/SCMTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/SCMTestUtils.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/SCMTestUtils.java
index a24f096..8e8e4f0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/SCMTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/SCMTestUtils.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.common;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto
@@ -117,7 +118,7 @@ public final class SCMTestUtils {
OzoneConfiguration conf = new OzoneConfiguration();
conf.set(HDDS_DATANODE_DIR_KEY, GenericTestUtils
.getRandomizedTempPath());
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, GenericTestUtils
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, GenericTestUtils
.getRandomizedTempPath());
return conf;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
index 3fc0dd0..260b158 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
@@ -20,6 +20,7 @@ import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ipc.RPC;
@@ -113,7 +114,7 @@ public class TestDatanodeStateMachine {
if (!dataDir.mkdirs()) {
LOG.info("Data dir create failed.");
}
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
new File(testRoot, "scm").getAbsolutePath());
path = Paths.get(path.toString(),
TestDatanodeStateMachine.class.getSimpleName() + ".id").toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index fea126b..2326818 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.ozone.container.ozoneimpl;
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
@@ -62,7 +62,7 @@ public class TestOzoneContainer {
conf = new OzoneConfiguration();
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, folder.getRoot()
.getAbsolutePath());
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
folder.newFolder().getAbsolutePath());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
index c6d85d8..fae6839 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
@@ -17,17 +17,18 @@
package org.apache.hadoop.hdds.server;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.http.client.methods.HttpRequestBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.InetSocketAddress;
+import java.util.Collection;
/**
* Generic utilities for all HDDS/Ozone servers.
@@ -116,6 +117,38 @@ public final class ServerUtils {
}
}
+ /**
+ * Get the location where SCM should store its metadata directories.
+ * Fall back to OZONE_METADATA_DIRS if not defined.
+ *
+ * @param conf
+ * @return
+ */
+ public static File getScmDbDir(Configuration conf) {
+ final Collection<String> metadirs = conf.getTrimmedStringCollection(
+ ScmConfigKeys.OZONE_SCM_DB_DIRS);
+
+ if (metadirs.size() > 1) {
+ throw new IllegalArgumentException(
+ "Bad config setting " + ScmConfigKeys.OZONE_SCM_DB_DIRS +
+ ". SCM does not support multiple metadata dirs currently");
+ }
+
+ if (metadirs.size() == 1) {
+ final File dbDirPath = new File(metadirs.iterator().next());
+ if (!dbDirPath.exists() && !dbDirPath.mkdirs()) {
+ throw new IllegalArgumentException("Unable to create directory " +
+ dbDirPath + " specified in configuration setting " +
+ ScmConfigKeys.OZONE_SCM_DB_DIRS);
+ }
+ return dbDirPath;
+ }
+
+ LOG.warn("{} is not configured. We recommend adding this setting. " +
+ "Falling back to {} instead.",
+ ScmConfigKeys.OZONE_SCM_DB_DIRS, HddsConfigKeys.OZONE_METADATA_DIRS);
+ return getOzoneMetaDirPath(conf);
+ }
/**
* Checks and creates Ozone Metadir Path if it does not exist.
@@ -125,9 +158,13 @@ public final class ServerUtils {
* @return File MetaDir
*/
public static File getOzoneMetaDirPath(Configuration conf) {
- String metaDirPath = conf.getTrimmed(OzoneConfigKeys
- .OZONE_METADATA_DIRS);
- Preconditions.checkNotNull(metaDirPath);
+ String metaDirPath = conf.getTrimmed(HddsConfigKeys.OZONE_METADATA_DIRS);
+
+ if (metaDirPath == null || metaDirPath.isEmpty()) {
+ throw new IllegalArgumentException(
+ HddsConfigKeys.OZONE_METADATA_DIRS + " must be defined.");
+ }
+
File dirPath = new File(metaDirPath);
if (!dirPath.exists() && !dirPath.mkdirs()) {
throw new IllegalArgumentException("Unable to create paths. Path: " +
@@ -138,7 +175,7 @@ public final class ServerUtils {
public static void setOzoneMetaDirPath(OzoneConfiguration conf,
String path) {
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, path);
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, path);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
index 2a8a3e3..a5ee130 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.scm.command
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -69,7 +70,6 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys
.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys
.OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
import static org.apache.hadoop.ozone.OzoneConsts.DELETED_BLOCK_DB;
/**
@@ -105,10 +105,10 @@ public class DeletedBlockLogImpl
maxRetry = conf.getInt(OZONE_SCM_BLOCK_DELETION_MAX_RETRY,
OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT);
- File metaDir = getOzoneMetaDirPath(conf);
- String scmMetaDataDir = metaDir.getPath();
- File deletedLogDbPath = new File(scmMetaDataDir, DELETED_BLOCK_DB);
- int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
+ final File metaDir = ServerUtils.getScmDbDir(conf);
+ final String scmMetaDataDir = metaDir.getPath();
+ final File deletedLogDbPath = new File(scmMetaDataDir, DELETED_BLOCK_DB);
+ final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
// Load store of all transactions.
deletedStore = MetadataStoreBuilder.newBuilder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
index 4e6f09e..0ea749f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.lease.Lease;
@@ -73,7 +74,6 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_D
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
.FAILED_TO_CHANGE_CONTAINER_STATE;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
/**
@@ -111,7 +111,7 @@ public class SCMContainerManager implements ContainerManager {
final NodeManager nodeManager, PipelineManager pipelineManager,
final EventPublisher eventPublisher) throws IOException {
- final File metaDir = getOzoneMetaDirPath(conf);
+ final File metaDir = ServerUtils.getScmDbDir(conf);
final File containerDBPath = new File(metaDir, SCM_CONTAINER_DB);
final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index a853693..1c21748 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.utils.MetadataStore;
@@ -46,7 +47,6 @@ import static org.apache.hadoop.hdds.scm
.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
import static org.apache.hadoop.hdds.scm
.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_PIPELINE_DB;
/**
@@ -74,8 +74,8 @@ public class SCMPipelineManager implements PipelineManager {
this.pipelineFactory = new PipelineFactory(nodeManager, stateManager, conf);
int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
- File metaDir = getOzoneMetaDirPath(conf);
- File pipelineDBPath = new File(metaDir, SCM_PIPELINE_DB);
+ final File metaDir = ServerUtils.getScmDbDir(conf);
+ final File pipelineDBPath = new File(metaDir, SCM_PIPELINE_DB);
this.pipelineStore =
MetadataStoreBuilder.newBuilder()
.setCreateIfMissing(true)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorage.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorage.java
index be6c1af..8a19850 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorage.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorage.java
@@ -19,13 +19,13 @@ package org.apache.hadoop.hdds.scm.server;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
+import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.ozone.common.Storage;
import java.io.IOException;
import java.util.Properties;
import java.util.UUID;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID;
import static org.apache.hadoop.ozone.OzoneConsts.STORAGE_DIR;
@@ -40,7 +40,7 @@ public class SCMStorage extends Storage {
* @throws IOException if any directories are inaccessible.
*/
public SCMStorage(OzoneConfiguration conf) throws IOException {
- super(NodeType.SCM, getOzoneMetaDirPath(conf), STORAGE_DIR);
+ super(NodeType.SCM, ServerUtils.getScmDbDir(conf), STORAGE_DIR);
}
public void setScmId(String scmId) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java
index 21acda8..6c4f249 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java
@@ -18,8 +18,12 @@
package org.apache.hadoop.hdds.scm;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.test.PathUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -27,6 +31,7 @@ import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.net.InetSocketAddress;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
@@ -34,6 +39,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRES
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Unit tests for {@link HddsServerUtil}
@@ -150,4 +156,48 @@ public class TestHddsServerUtils {
thrown.expect(IllegalArgumentException.class);
HddsServerUtil.getScmAddressForDataNodes(conf);
}
+
+ /**
+ * Test {@link ServerUtils#getScmDbDir}.
+ */
+ @Test
+ public void testGetScmDbDir() {
+ final File testDir = PathUtils.getTestDir(TestHddsServerUtils.class);
+ final File dbDir = new File(testDir, "scmDbDir");
+ final File metaDir = new File(testDir, "metaDir"); // should be ignored.
+ final Configuration conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, dbDir.getPath());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
+
+ try {
+ assertEquals(dbDir, ServerUtils.getScmDbDir(conf));
+ assertTrue(dbDir.exists()); // should have been created.
+ } finally {
+ FileUtils.deleteQuietly(dbDir);
+ }
+ }
+
+ /**
+ * Test {@link ServerUtils#getScmDbDir} with fallback to OZONE_METADATA_DIRS
+ * when OZONE_SCM_DB_DIRS is undefined.
+ */
+ @Test
+ public void testGetScmDbDirWithFallback() {
+ final File testDir = PathUtils.getTestDir(TestHddsServerUtils.class);
+ final File metaDir = new File(testDir, "metaDir");
+ final Configuration conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
+ try {
+ assertEquals(metaDir, ServerUtils.getScmDbDir(conf));
+ assertTrue(metaDir.exists()); // should have been created.
+ } finally {
+ FileUtils.deleteQuietly(metaDir);
+ }
+ }
+
+ @Test
+ public void testNoScmDbDirConfigured() {
+ thrown.expect(IllegalArgumentException.class);
+ ServerUtils.getScmDbDir(new OzoneConfiguration());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
index 32e8640..aa940df 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.scm.block;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
@@ -34,7 +35,6 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.test.GenericTestUtils;
@@ -79,7 +79,7 @@ public class TestBlockManager implements EventHandler<Boolean> {
String path = GenericTestUtils
.getTempPath(TestBlockManager.class.getSimpleName());
testDir = Paths.get(path).toFile();
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, path);
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, path);
eventQueue = new EventQueue();
boolean folderExisted = testDir.exists() || testDir.mkdirs();
if (!folderExisted) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
index 06f4f5e..268cf8b 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.block;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
@@ -60,7 +61,6 @@ import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys
.OZONE_SCM_BLOCK_DELETION_MAX_RETRY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.when;
@@ -81,7 +81,7 @@ public class TestDeletedBlockLog {
TestDeletedBlockLog.class.getSimpleName());
conf = new OzoneConfiguration();
conf.setInt(OZONE_SCM_BLOCK_DELETION_MAX_RETRY, 20);
- conf.set(OZONE_METADATA_DIRS, testDir.getAbsolutePath());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
containerManager = Mockito.mock(SCMContainerManager.class);
deletedBlockLog = new DeletedBlockLogImpl(conf, containerManager);
dnList = new ArrayList<>(3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
index 44a8deb..b1d24d5 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
@@ -21,6 +21,7 @@ import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers
@@ -28,7 +29,6 @@ import org.apache.hadoop.hdds.scm.container.common.helpers
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
@@ -65,7 +65,7 @@ public class TestCloseContainerEventHandler {
testDir = GenericTestUtils
.getTestDir(TestCloseContainerEventHandler.class.getSimpleName());
configuration
- .set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
+ .set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
nodeManager = new MockNodeManager(true, 10);
PipelineManager pipelineManager =
new SCMPipelineManager(configuration, nodeManager, eventQueue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
index a5475e2..faae6f5 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -40,7 +41,6 @@ import org.apache.hadoop.hdds.server.events.Event;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Before;
@@ -74,7 +74,7 @@ public class TestContainerReportHandler implements EventPublisher {
this.getClass().getSimpleName());
//GIVEN
OzoneConfiguration conf = new OzoneConfiguration();
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir);
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir);
EventQueue eventQueue = new EventQueue();
PipelineManager pipelineManager =
new SCMPipelineManager(conf, nodeManager, eventQueue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
index 02c292c..be4cce6 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.container;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -34,7 +35,6 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
@@ -77,7 +77,7 @@ public class TestSCMContainerManager {
testDir = GenericTestUtils
.getTestDir(TestSCMContainerManager.class.getSimpleName());
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
testDir.getAbsolutePath());
conf.setTimeDuration(
ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
index e283732..7a9dbad 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdds.scm.node;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
@@ -36,7 +37,6 @@ import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.test.PathUtils;
import org.junit.Ignore;
@@ -130,7 +130,7 @@ public class TestContainerPlacement {
final File testDir = PathUtils.getTestDir(
TestContainerPlacement.class);
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
testDir.getAbsolutePath());
conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 985fa2c..51099f4 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@ -26,6 +26,7 @@ import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -48,7 +49,6 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeRepo
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
@@ -74,7 +74,7 @@ public class TestDeadNodeHandler {
OzoneConfiguration conf = new OzoneConfiguration();
storageDir = GenericTestUtils.getTempPath(
TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID());
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, storageDir);
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
eventQueue = new EventQueue();
nodeManager = new SCMNodeManager(conf, "cluster1", null, eventQueue);
PipelineManager pipelineManager =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index c899bda..08cbdd7 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
@@ -30,7 +31,6 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
@@ -104,7 +104,7 @@ public class TestNodeManager {
*/
OzoneConfiguration getConf() {
OzoneConfiguration conf = new OzoneConfiguration();
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
testDir.getAbsolutePath());
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
TimeUnit.MILLISECONDS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index b2ddf39..3a2ce78 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -75,6 +75,8 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static org.mockito.Mockito.mock;
import java.io.File;
@@ -82,7 +84,6 @@ import java.net.InetSocketAddress;
import java.util.UUID;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils
.createEndpoint;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 0974104..d8608be 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -17,12 +17,20 @@
package org.apache.hadoop.ozone;
+import java.io.File;
import java.net.InetSocketAddress;
+import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.net.NetUtils;
import com.google.common.base.Optional;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
@@ -36,6 +44,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_PORT_DEFAULT;
* communication.
*/
public final class OmUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(OmUtils.class);
private OmUtils() {
}
@@ -91,4 +100,37 @@ public final class OmUtils {
getPortNumberFromConfigKeys(conf, OZONE_OM_HTTP_ADDRESS_KEY);
return port.or(OZONE_OM_HTTP_BIND_PORT_DEFAULT);
}
+
+ /**
+ * Get the location where OM should store its metadata directories.
+ * Fall back to OZONE_METADATA_DIRS if not defined.
+ *
+ * @param conf
+ * @return
+ */
+ public static File getOmDbDir(Configuration conf) {
+ final Collection<String> dbDirs = conf.getTrimmedStringCollection(
+ OMConfigKeys.OZONE_OM_DB_DIRS);
+
+ if (dbDirs.size() > 1) {
+ throw new IllegalArgumentException(
+ "Bad configuration setting " + OMConfigKeys.OZONE_OM_DB_DIRS +
+ ". OM does not support multiple metadata dirs currently.");
+ }
+
+ if (dbDirs.size() == 1) {
+ final File dbDirPath = new File(dbDirs.iterator().next());
+ if (!dbDirPath.exists() && !dbDirPath.mkdirs()) {
+ throw new IllegalArgumentException("Unable to create directory " +
+ dbDirPath + " specified in configuration setting " +
+ OMConfigKeys.OZONE_OM_DB_DIRS);
+ }
+ return dbDirPath;
+ }
+
+ LOG.warn("{} is not configured. We recommend adding this setting. " +
+ "Falling back to {} instead.",
+ OMConfigKeys.OZONE_OM_DB_DIRS, HddsConfigKeys.OZONE_METADATA_DIRS);
+ return ServerUtils.getOzoneMetaDirPath(conf);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index b9ca296..6e9acb4 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -28,6 +28,9 @@ public final class OMConfigKeys {
private OMConfigKeys() {
}
+ // Location where the OM stores its DB files. In the future we may support
+ // multiple entries for performance (sharding)..
+ public static final String OZONE_OM_DB_DIRS = "ozone.om.db.dirs";
public static final String OZONE_OM_HANDLER_COUNT_KEY =
"ozone.om.handler.count.key";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
new file mode 100644
index 0000000..2001598
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.test.PathUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
+
+import java.io.File;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit tests for {@link OmUtils}.
+ */
+public class TestOmUtils {
+ @Rule
+ public Timeout timeout = new Timeout(60_000);
+
+ @Rule
+ public ExpectedException thrown= ExpectedException.none();
+
+ /**
+ * Test {@link OmUtils#getOmDbDir}.
+ */
+ @Test
+ public void testGetOmDbDir() {
+ final File testDir = PathUtils.getTestDir(TestOmUtils.class);
+ final File dbDir = new File(testDir, "omDbDir");
+ final File metaDir = new File(testDir, "metaDir"); // should be ignored.
+ final Configuration conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, dbDir.getPath());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
+
+ try {
+ assertEquals(dbDir, OmUtils.getOmDbDir(conf));
+ assertTrue(dbDir.exists()); // should have been created.
+ } finally {
+ FileUtils.deleteQuietly(dbDir);
+ }
+ }
+
+ /**
+ * Test {@link OmUtils#getOmDbDir} with fallback to OZONE_METADATA_DIRS
+ * when OZONE_OM_DB_DIRS is undefined.
+ */
+ @Test
+ public void testGetOmDbDirWithFallback() {
+ final File testDir = PathUtils.getTestDir(TestOmUtils.class);
+ final File metaDir = new File(testDir, "metaDir");
+ final Configuration conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
+
+ try {
+ assertEquals(metaDir, OmUtils.getOmDbDir(conf));
+ assertTrue(metaDir.exists()); // should have been created.
+ } finally {
+ FileUtils.deleteQuietly(metaDir);
+ }
+ }
+
+ @Test
+ public void testNoOmDbDirConfigured() {
+ thrown.expect(IllegalArgumentException.class);
+ OmUtils.getOmDbDir(new OzoneConfiguration());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
index dab7fb6..4487766 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.scm.pipeline;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -28,7 +29,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -54,7 +54,7 @@ public class TestSCMPipelineManager {
conf = new OzoneConfiguration();
testDir = GenericTestUtils
.getTestDir(TestSCMPipelineManager.class.getSimpleName());
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
boolean folderExisted = testDir.exists() || testDir.mkdirs();
if (!folderExisted) {
throw new IOException("Unable to create test directory path");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 6c0f408..37b6fdc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -24,6 +24,7 @@ import java.util.Optional;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -389,7 +390,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, ozoneEnabled);
Path metaDir = Paths.get(path, "ozone-meta");
Files.createDirectories(metaDir);
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
configureTrace();
}
@@ -466,7 +467,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
Files.createDirectories(metaDir);
Files.createDirectories(dataDir);
Files.createDirectories(ratisDir);
- dnConf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
+ dnConf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataDir.toString());
dnConf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR,
ratisDir.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index e616eef..0ca4749 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
@@ -47,7 +48,6 @@ import java.util.List;
import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
import static org.junit.Assert.*;
/**
@@ -65,8 +65,7 @@ public class TestMiniOzoneCluster {
@BeforeClass
public static void setup() {
conf = new OzoneConfiguration();
- conf.set(OZONE_METADATA_DIRS,
- TEST_ROOT.toString());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, TEST_ROOT.toString());
conf.setBoolean(DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, true);
WRITE_TMP.mkdirs();
READ_TMP.mkdirs();
@@ -156,7 +155,7 @@ public class TestMiniOzoneCluster {
Configuration ozoneConf = SCMTestUtils.getConf();
File testDir = PathUtils.getTestDir(TestOzoneContainer.class);
ozoneConf.set(DFS_DATANODE_DATA_DIR_KEY, testDir.getAbsolutePath());
- ozoneConf.set(OZONE_METADATA_DIRS,
+ ozoneConf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
TEST_ROOT.toString());
// Each instance of SM will create an ozone container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 52340a9..23d371d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -34,6 +34,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
@@ -388,7 +389,7 @@ public class TestStorageContainerManager {
final String path = GenericTestUtils.getTempPath(
UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
StartupOption.INIT.setClusterId("testClusterId");
// This will initialize SCM
@@ -410,7 +411,7 @@ public class TestStorageContainerManager {
final String path = GenericTestUtils.getTempPath(
UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
//This will set the cluster id in the version file
MiniOzoneCluster cluster =
MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
@@ -430,7 +431,7 @@ public class TestStorageContainerManager {
final String path =
GenericTestUtils.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
exception.expect(SCMException.class);
exception.expectMessage("SCM not initialized.");
@@ -455,7 +456,7 @@ public class TestStorageContainerManager {
final String path =
GenericTestUtils.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
SCMStorage scmStore = new SCMStorage(conf);
String clusterId = UUID.randomUUID().toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
index 44df68a..99c07ff 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -1323,7 +1324,7 @@ public class TestOzoneManager {
final String path =
GenericTestUtils.getTempPath(UUID.randomUUID().toString());
Path metaDirPath = Paths.get(path, "om-meta");
- config.set(OzoneConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
+ config.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
config.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
config.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
config.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
index 3820aed..b7b6929 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
@@ -22,11 +22,12 @@ import java.util.Properties;
import java.util.UUID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
/**
* OMStorage is responsible for management of the StorageDirectories used by
@@ -42,7 +43,7 @@ public class OMStorage extends Storage {
* @throws IOException if any directories are inaccessible.
*/
public OMStorage(OzoneConfiguration conf) throws IOException {
- super(NodeType.OM, getOzoneMetaDirPath(conf), STORAGE_DIR);
+ super(NodeType.OM, OmUtils.getOmDbDir(conf), STORAGE_DIR);
}
public void setScmId(String scmId) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 75bd712..cb7ca1f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -22,7 +22,9 @@ import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -54,7 +56,6 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
@@ -114,7 +115,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
private final Table s3Table;
public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException {
- File metaDir = getOzoneMetaDirPath(conf);
+ File metaDir = OmUtils.getOmDbDir(conf);
this.lock = new OzoneManagerLock(conf);
this.openKeyExpireThresholdMS = 1000 * conf.getInt(
OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
index a3ff6c8..1dc3cab 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.ozone.om;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -171,7 +172,7 @@ public class TestOmSQLCli {
String dbOutPath = GenericTestUtils.getTempPath(
UUID.randomUUID() + "/out_sql.db");
- String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
+ String dbRootPath = conf.get(HddsConfigKeys.OZONE_METADATA_DIRS);
String dbPath = dbRootPath + "/" + OM_DB_NAME;
String[] args = {"-p", dbPath, "-o", dbOutPath};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08bb0362/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index 054e668..3247b87 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.scm;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -192,7 +193,7 @@ public class TestContainerSQLCli {
UUID.randomUUID() + "/out_sql.db");
// TODO : the following will fail due to empty Datanode list, need to fix.
//String dnUUID = cluster.getDataNodes().get(0).getUuid();
- String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
+ String dbRootPath = conf.get(HddsConfigKeys.OZONE_METADATA_DIRS);
String dbPath = dbRootPath + "/" + SCM_CONTAINER_DB;
String[] args = {"-p", dbPath, "-o", dbOutPath};
Connection conn;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDFS-13942. [JDK10] Fix javadoc
errors in hadoop-hdfs module. Contributed by Dinesh Chitlangia.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java
index 5e708be..a195bf1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java
@@ -47,7 +47,7 @@ public enum Quota {
/**
* Is quota violated?
- * The quota is violated if quota is set and usage > quota.
+ * The quota is violated if quota is set and usage > quota.
*/
public static boolean isViolated(final long quota, final long usage) {
return quota >= 0 && usage > quota;
@@ -55,7 +55,8 @@ public enum Quota {
/**
* Is quota violated?
- * The quota is violated if quota is set, delta > 0 and usage + delta > quota.
+ * The quota is violated if quota is set, delta > 0 and
+ * usage + delta > quota.
*/
static boolean isViolated(final long quota, final long usage,
final long delta) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
index a8acccd..2e13df5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
@@ -319,7 +319,7 @@ public class ReencryptionHandler implements Runnable {
/**
* Main loop. It takes at most 1 zone per scan, and executes until the zone
* is completed.
- * {@see #reencryptEncryptionZoneInt(Long)}.
+ * {@link #reencryptEncryptionZone(long)}.
*/
@Override
public void run() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
index e1bf027..b6f4f64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
@@ -31,7 +31,7 @@ import com.google.common.base.Preconditions;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
/**
- * There are four types of extended attributes <XAttr> defined by the
+ * There are four types of extended attributes <XAttr> defined by the
* following namespaces:
* <br>
* USER - extended user attributes: these can be assigned to files and
@@ -56,7 +56,7 @@ import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_
* is called on a file or directory in the /.reserved/raw HDFS directory
* hierarchy. These attributes can only be accessed by the user who have
* read access.
- * </br>
+ * <br>
*/
@InterfaceAudience.Private
public class XAttrPermissionFilter {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
index 1dab69c..d856f6d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
@@ -33,7 +33,7 @@ public class XAttrStorage {
/**
* Reads the extended attribute of an inode by name with prefix.
- * <p/>
+ * <p>
*
* @param inode INode to read
* @param snapshotId the snapshotId of the requested path
@@ -48,11 +48,11 @@ public class XAttrStorage {
/**
* Reads the existing extended attributes of an inode.
- * <p/>
+ * <p>
* Must be called while holding the FSDirectory read lock.
*
* @param inodeAttr INodeAttributes to read.
- * @return List<XAttr> <code>XAttr</code> list.
+ * @return {@code XAttr} list.
*/
public static List<XAttr> readINodeXAttrs(INodeAttributes inodeAttr) {
XAttrFeature f = inodeAttr.getXAttrFeature();
@@ -61,7 +61,7 @@ public class XAttrStorage {
/**
* Update xattrs of inode.
- * <p/>
+ * <p>
* Must be called while holding the FSDirectory write lock.
*
* @param inode INode to update
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
index d115656..1ba59a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
@@ -157,10 +157,10 @@ abstract class AbstractINodeDiffList<N extends INode,
/**
* Find the latest snapshot before a given snapshot.
- * @param anchorId The returned snapshot's id must be <= or < this given
- * snapshot id.
- * @param exclusive True means the returned snapshot's id must be < the given
- * id, otherwise <=.
+ * @param anchorId The returned snapshot's id must be <= or < this
+ * given snapshot id.
+ * @param exclusive True means the returned snapshot's id must be < the
+ * given id, otherwise <=.
* @return The id of the latest snapshot before the given snapshot.
*/
public final int getPrior(int anchorId, boolean exclusive) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java
index 85d9a6d..705b8d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java
@@ -44,15 +44,15 @@ import java.util.Objects;
* and created after a particular snapshot and before the next snapshot. The
* sequence will look like this:
* <p>
- * s0->s1->s2->s3->s4->s5->s6->s7->s8->s9.
+ * {@literal s0->s1->s2->s3->s4->s5->s6->s7->s8->s9}.
* <p>
* Assuming a skip interval of 3, which means a new diff will be added at a
* level higher than the current level after we have ore than 3 snapshots.
* Next level promotion happens after 9 snapshots and so on.
* <p>
- * level 2: s08------------------------------->s9
- * level 1: S02------->s35-------->s68-------->s9
- * level 0: s0->s1->s2->s3->s4->s5->s6->s7->s8->s9
+ * level 2: {@literal s08------------------------------->s9}
+ * level 1: {@literal S02------->s35-------->s68-------->s9}
+ * level 0: {@literal s0->s1->s2->s3->s4->s5->s6->s7->s8->s9}
* <p>
* s02 will be created by combining diffs for s0, s1, s2 once s3 gets created.
* Similarly, s08 will be created by combining s02, s35 and s68 once s9 gets
@@ -143,6 +143,7 @@ public class DiffListBySkipList implements DiffList<DirectoryDiff> {
* and level.
*
* @param diff The element to be stored in the node.
+ * @param level
*/
SkipListNode(DirectoryDiff diff, int level) {
this.diff = diff;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java
index 02b9cff..b3f8de9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java
@@ -106,8 +106,8 @@ public class BlockStorageMovementNeeded {
* Add the itemInfo to tracking list for which storage movement expected if
* necessary.
*
- * @param itemInfoList
- * - List of child in the directory
+ * @param itemInfo
+ * - child in the directory
* @param scanCompleted
* -Indicates whether the ItemInfo start id directory has no more
* elements to scan.
@@ -191,7 +191,6 @@ public class BlockStorageMovementNeeded {
/**
* Clean all the movements in spsDirsToBeTraveresed/storageMovementNeeded
* and notify to clean up required resources.
- * @throws IOException
*/
public synchronized void clearQueuesWithNotification() {
// Remove xAttr from directories
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/DatanodeCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/DatanodeCacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/DatanodeCacheManager.java
index d4e514b..be8d01f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/DatanodeCacheManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/DatanodeCacheManager.java
@@ -69,7 +69,7 @@ public class DatanodeCacheManager {
/**
* Returns the live datanodes and its storage details, which has available
- * space (> 0) to schedule block moves. This will return array of datanodes
+ * space (> 0) to schedule block moves. This will return array of datanodes
* from its local cache. It has a configurable refresh interval in millis and
* periodically refresh the datanode cache by fetching latest
* {@link Context#getLiveDatanodeStorageReport()} once it elapsed refresh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java
index 074eab6..14cf05d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java
@@ -39,7 +39,7 @@ import com.google.common.annotations.VisibleForTesting;
* configured by the administrator.
*
* <p>
- * If the configured mode is {@link StoragePolicySatisfierMode.EXTERNAL}, then
+ * If the configured mode is {@link StoragePolicySatisfierMode#EXTERNAL}, then
* it won't do anything, just maintains the sps invoked path ids. Administrator
* requires to start external sps service explicitly, to fetch the sps invoked
* path ids from namenode, then do necessary computations and block movement in
@@ -48,7 +48,7 @@ import com.google.common.annotations.VisibleForTesting;
* external sps service functionality.
*
* <p>
- * If the configured mode is {@link StoragePolicySatisfierMode.NONE}, then it
+ * If the configured mode is {@link StoragePolicySatisfierMode#NONE}, then it
* will disable the sps feature completely by clearing all queued up sps path's
* hint.
*
@@ -88,12 +88,12 @@ public class StoragePolicySatisfyManager {
* This function will do following logic based on the configured sps mode:
*
* <p>
- * If the configured mode is {@link StoragePolicySatisfierMode.EXTERNAL}, then
+ * If the configured mode is {@link StoragePolicySatisfierMode#EXTERNAL}, then
* it won't do anything. Administrator requires to start external sps service
* explicitly.
*
* <p>
- * If the configured mode is {@link StoragePolicySatisfierMode.NONE}, then the
+ * If the configured mode is {@link StoragePolicySatisfierMode#NONE}, then the
* service is disabled and won't do any action.
*/
public void start() {
@@ -121,12 +121,12 @@ public class StoragePolicySatisfyManager {
* This function will do following logic based on the configured sps mode:
*
* <p>
- * If the configured mode is {@link StoragePolicySatisfierMode.EXTERNAL}, then
+ * If the configured mode is {@link StoragePolicySatisfierMode#EXTERNAL}, then
* it won't do anything. Administrator requires to stop external sps service
* explicitly, if needed.
*
* <p>
- * If the configured mode is {@link StoragePolicySatisfierMode.NONE}, then the
+ * If the configured mode is {@link StoragePolicySatisfierMode#NONE}, then the
* service is disabled and won't do any action.
*/
public void stop() {
@@ -225,6 +225,7 @@ public class StoragePolicySatisfyManager {
/**
* Verify that satisfier queue limit exceeds allowed outstanding limit.
+ * @throws IOException
*/
public void verifyOutstandingPathQLimit() throws IOException {
long size = pathsToBeTraveresed.size();
@@ -269,6 +270,7 @@ public class StoragePolicySatisfyManager {
/**
* Adds the sps path to SPSPathIds list.
+ * @param id
*/
public void addPathId(long id) {
synchronized (pathsToBeTraveresed) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressView.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressView.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressView.java
index 3101741..a4453a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressView.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StartupProgressView.java
@@ -179,7 +179,7 @@ public class StartupProgressView {
/**
* Returns all phases.
*
- * @return Iterable<Phase> containing all phases
+ * @return {@code Iterable<Phase>} containing all phases
*/
public Iterable<Phase> getPhases() {
return EnumSet.allOf(Phase.class);
@@ -189,7 +189,7 @@ public class StartupProgressView {
* Returns all steps within a phase.
*
* @param phase Phase to get
- * @return Iterable<Step> all steps
+ * @return {@code Iterable<Step>} all steps
*/
public Iterable<Step> getSteps(Phase phase) {
return new TreeSet<Step>(phases.get(phase).steps.keySet());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
index 4d61d0f..72ec9f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
@@ -47,22 +47,22 @@ import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowMan
/**
* The interface to the top metrics.
- * <p/>
+ * <p>
* Metrics are collected by a custom audit logger, {@link org.apache.hadoop
* .hdfs.server.namenode.top.TopAuditLogger}, which calls TopMetrics to
* increment per-operation, per-user counts on every audit log call. These
* counts are used to show the top users by NameNode operation as well as
* across all operations.
- * <p/>
+ * <p>
* TopMetrics maintains these counts for a configurable number of time
* intervals, e.g. 1min, 5min, 25min. Each interval is tracked by a
* RollingWindowManager.
- * <p/>
+ * <p>
* These metrics are published as a JSON string via {@link org.apache.hadoop
* .hdfs.server .namenode.metrics.FSNamesystemMBean#getTopWindows}. This is
* done by calling {@link org.apache.hadoop.hdfs.server.namenode.top.window
* .RollingWindowManager#snapshot} on each RollingWindowManager.
- * <p/>
+ * <p>
* Thread-safe: relies on thread-safety of RollingWindowManager
*/
@InterfaceAudience.Private
@@ -119,6 +119,13 @@ public class TopMetrics implements MetricsSource {
* log file. This is to be consistent when {@link TopMetrics} is charged with
* data read back from log files instead of being invoked directly by the
* FsNamesystem
+ * @param succeeded
+ * @param userName
+ * @param addr
+ * @param cmd
+ * @param src
+ * @param dst
+ * @param status
*/
public void report(boolean succeeded, String userName, InetAddress addr,
String cmd, String src, String dst, FileStatus status) {
@@ -147,6 +154,8 @@ public class TopMetrics implements MetricsSource {
* {@link org.apache.hadoop.metrics2.MetricsRecord}s for consumption by
* external metrics systems. Each metrics record added corresponds to the
* reporting period a.k.a window length of the configured rolling windows.
+ * @param collector
+ * @param all
*/
@Override
public void getMetrics(MetricsCollector collector, boolean all) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindow.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindow.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindow.java
index 63ff125..f927106 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindow.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindow.java
@@ -29,23 +29,24 @@ import org.slf4j.LoggerFactory;
* Events are reported based on occurrence time. The total number of events in
* the last period covered by the rolling window can be retrieved by the
* {@link #getSum(long)} method.
- * <p/>
+ * <p>
*
* Assumptions:
- * <p/>
+ * <p>
*
* (1) Concurrent invocation of {@link #incAt} method are possible
- * <p/>
+ * <p>
*
* (2) The time parameter of two consecutive invocation of {@link #incAt} could
* be in any given order
- * <p/>
+ * <p>
*
* (3) The buffering delays are not more than the window length, i.e., after two
* consecutive invocation {@link #incAt(long time1, long)} and
- * {@link #incAt(long time2, long)}, time1 < time2 || time1 - time2 < windowLenMs.
+ * {@link #incAt(long time2, long)}, time1 < time2 || time1 - time2 <
+ * windowLenMs.
* This assumption helps avoiding unnecessary synchronizations.
- * <p/>
+ * <p>
*
* Thread-safety is built in the {@link RollingWindow.Bucket}
*/
@@ -85,7 +86,7 @@ public class RollingWindow {
/**
* When an event occurs at the specified time, this method reflects that in
* the rolling window.
- * <p/>
+ * <p>
*
* @param time the time at which the event occurred
* @param delta the delta that will be added to the window
@@ -153,6 +154,7 @@ public class RollingWindow {
* performed. We do not need to update the {@link #updateTime} because as
* long as the {@link #updateTime} belongs to the current view of the
* rolling window, the algorithm works fine.
+ * @param delta
*/
void inc(long delta) {
value.addAndGet(delta);
@@ -161,7 +163,7 @@ public class RollingWindow {
/**
* Get value represented by this window at the specified time
- * <p/>
+ * <p>
*
* If time lags behind the latest update time, the new updates are still
* included in the sum
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
index bdd0ab0..095294e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
@@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory;
* A class to manage the set of {@link RollingWindow}s. This class is the
* interface of metrics system to the {@link RollingWindow}s to retrieve the
* current top metrics.
- * <p/>
+ * <p>
* Thread-safety is provided by each {@link RollingWindow} being thread-safe as
* well as {@link ConcurrentHashMap} for the collection of them.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockStorageMovementCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockStorageMovementCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockStorageMovementCommand.java
index e90317d..6b0b261 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockStorageMovementCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockStorageMovementCommand.java
@@ -30,11 +30,12 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
*
* Upon receiving this command, this DataNode pass the array of block movement
* details to
- * {@link org.apache.hadoop.hdfs.server.datanode.StoragePolicySatisfyWorker}
- * service. Later, StoragePolicySatisfyWorker will schedule block movement tasks
- * for these blocks and monitors the completion of each task. After the block
- * movement attempt is finished(with success or failure) this DataNode will send
- * response back to NameNode about the block movement attempt finished details.
+ * {@link org.apache.hadoop.hdfs.server.sps.ExternalSPSBlockMoveTaskHandler}
+ * service. Later, ExternalSPSBlockMoveTaskHandler will schedule block movement
+ * tasks for these blocks and monitors the completion of each task. After the
+ * block movement attempt is finished(with success or failure) this DataNode
+ * will send response back to NameNode about the block movement attempt
+ * finished details.
*/
public class BlockStorageMovementCommand extends DatanodeCommand {
private final String blockPoolId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
index 311b68f..5680ef3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
@@ -158,7 +158,7 @@ public interface DatanodeProtocol {
* {@link #blockReport(DatanodeRegistration, String, StorageBlockReport[], BlockReportContext)},
* which is used to communicated blocks stored on disk.
*
- * @param The datanode registration.
+ * @param registration The datanode registration.
* @param poolId The block pool ID for the blocks.
* @param blockIds A list of block IDs.
* @return The DatanodeCommand.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
index f80477b..5d609de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
@@ -73,7 +73,7 @@ public interface NamenodeProtocol {
* @param datanode a data node
* @param size requested size
* @param minBlockSize each block should be of this minimum Block Size
- * @return a list of blocks & their locations
+ * @return BlocksWithLocations a list of blocks & their locations
* @throws IOException if size is less than or equal to 0 or
datanode does not exist
*/
@@ -183,7 +183,8 @@ public interface NamenodeProtocol {
/**
* Return a structure containing details about all edit logs
* available to be fetched from the NameNode.
- * @param sinceTxId return only logs that contain transactions >= sinceTxId
+ * @param sinceTxId return only logs that contain transactions {@literal >=}
+ * sinceTxId
*/
@Idempotent
public RemoteEditLogManifest getEditLogManifest(long sinceTxId)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java
index 3ea0294..64dec8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java
@@ -36,8 +36,10 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 3171497..2afc97c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -50,14 +50,17 @@ import org.apache.hadoop.util.ToolRunner;
* <p>The tool scans all files and directories, starting from an indicated
* root path. The following abnormal conditions are detected and handled:</p>
* <ul>
- * <li>files with blocks that are completely missing from all datanodes.<br/>
+ * <li>files with blocks that are completely missing from all datanodes.<br>
* In this case the tool can perform one of the following actions:
* <ul>
- * <li>none ({@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#FIXING_NONE})</li>
* <li>move corrupted files to /lost+found directory on DFS
- * ({@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#FIXING_MOVE}). Remaining data blocks are saved as a
- * block chains, representing longest consecutive series of valid blocks.</li>
- * <li>delete corrupted files ({@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#FIXING_DELETE})</li>
+ * ({@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#doMove}).
+ * Remaining data blocks are saved as a
+ * block chains, representing longest consecutive series of valid blocks.
+ * </li>
+ * <li>delete corrupted files
+ * ({@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#doDelete})
+ * </li>
* </ul>
* </li>
* <li>detect files with under-replicated or over-replicated blocks</li>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
index f075ed2..43eedf8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
@@ -121,8 +121,8 @@ public class OfflineEditsViewer extends Configured implements Tool {
/** Process an edit log using the chosen processor or visitor.
*
- * @param inputFilename The file to process
- * @param outputFilename The output file name
+ * @param inputFileName The file to process
+ * @param outputFileName The output file name
* @param processor If visitor is null, the processor to use
* @param visitor If non-null, the visitor to use.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsVisitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsVisitor.java
index eb477e1..1383f4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsVisitor.java
@@ -52,7 +52,7 @@ abstract public interface OfflineEditsVisitor {
* Begin visiting an element that encloses another element, such as
* the beginning of the list of blocks that comprise a file.
*
- * @param value Token being visited
+ * @param op Token being visited
*/
abstract void visitOp(FSEditLogOp op)
throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java
index c84e2ed..cc97ea7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java
@@ -50,9 +50,7 @@ public class StatisticsEditsVisitor implements OfflineEditsVisitor {
* Create a processor that writes to the file named and may or may not
* also output to the screen, as specified.
*
- * @param filename Name of file to write output to
- * @param tokenizer Input tokenizer
- * @param printToScreen Mirror output to screen?
+ * @param out Name of file to write output to
*/
public StatisticsEditsVisitor(OutputStream out) throws IOException {
this.out = new PrintWriter(new OutputStreamWriter(out, Charsets.UTF_8));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
index 28bcf10..9c7b7f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
@@ -26,11 +26,13 @@ import org.apache.hadoop.classification.InterfaceAudience;
/**
* File name distribution visitor.
* <p>
- * It analyzes file names in fsimage and prints the following information:
+ * It analyzes file names in fsimage and prints the following information:
+ * <ul>
* <li>Number of unique file names</li>
* <li>Number file names and the corresponding number range of files that use
* these same names</li>
* <li>Heap saved if the file name objects are reused</li>
+ * </ul>
*/
@InterfaceAudience.Private
public class NameDistributionVisitor extends TextWriterImageVisitor {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java
index 1f87a7a..188537b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java
@@ -470,23 +470,23 @@ public class Diff<K, E extends Diff.Element<K>> {
* <pre>
* 1. For (c, 0) in the posterior diff, check the element in this diff:
* 1.1 (c', 0) in this diff: impossible
- * 1.2 (0, d') in this diff: put in c-list --> (c, d')
+ * 1.2 (0, d') in this diff: put in c-list --> (c, d')
* 1.3 (c', d') in this diff: impossible
- * 1.4 (0, 0) in this diff: put in c-list --> (c, 0)
+ * 1.4 (0, 0) in this diff: put in c-list --> (c, 0)
* This is the same logic as create(E).
*
* 2. For (0, d) in the posterior diff,
- * 2.1 (c', 0) in this diff: remove from c-list --> (0, 0)
+ * 2.1 (c', 0) in this diff: remove from c-list --> (0, 0)
* 2.2 (0, d') in this diff: impossible
- * 2.3 (c', d') in this diff: remove from c-list --> (0, d')
- * 2.4 (0, 0) in this diff: put in d-list --> (0, d)
+ * 2.3 (c', d') in this diff: remove from c-list --> (0, d')
+ * 2.4 (0, 0) in this diff: put in d-list --> (0, d)
* This is the same logic as delete(E).
*
* 3. For (c, d) in the posterior diff,
- * 3.1 (c', 0) in this diff: replace the element in c-list --> (c, 0)
+ * 3.1 (c', 0) in this diff: replace the element in c-list --> (c, 0)
* 3.2 (0, d') in this diff: impossible
- * 3.3 (c', d') in this diff: replace the element in c-list --> (c, d')
- * 3.4 (0, 0) in this diff: put in c-list and d-list --> (c, d)
+ * 3.3 (c', d') in this diff: replace the element in c-list --> (c, d')
+ * 3.4 (0, 0) in this diff: put in c-list and d-list --> (c, d)
* This is the same logic as modify(E, E).
* </pre>
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac9f91b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java
index f23b021..8c783d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java
@@ -116,8 +116,8 @@ public class XMLUtils {
*
* There are three kinds of code points in XML:
* - Those that can be represented normally,
- * - Those that have to be escaped (for example, & must be represented
- * as &)
+ * - Those that have to be escaped (for example, & must be represented
+ * as {@literal &})
* - Those that cannot be represented at all in XML.
*
* The built-in SAX functions will handle the first two types for us just
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDDS-4
Posted by ae...@apache.org.
Merge branch 'trunk' into HDDS-4
Conflicts:
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7119be30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7119be30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7119be30
Branch: refs/heads/HDDS-4
Commit: 7119be30bd570e65c97ff88e8aa84705c79f227b
Parents: 2115256 f3f5e7a
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Nov 5 12:37:32 2018 -0800
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Nov 5 12:37:32 2018 -0800
----------------------------------------------------------------------
LICENSE.txt | 5 +-
NOTICE.txt | 8 +-
dev-support/bin/dist-layout-stitching | 1 +
dev-support/docker/Dockerfile | 1 +
.../assemblies/hadoop-registry-dist.xml | 41 +
.../hadoop-client-minicluster/pom.xml | 2 +-
.../hadoop-common/src/main/bin/hadoop | 6 +
.../hadoop-common/src/main/conf/hadoop-env.sh | 13 +
.../fs/CommonConfigurationKeysPublic.java | 31 +-
.../io/compress/zstd/ZStandardDecompressor.java | 4 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 114 +-
.../org/apache/hadoop/security/Credentials.java | 15 +
.../hadoop/security/SaslPropertiesResolver.java | 4 +-
.../hadoop/security/UserGroupInformation.java | 192 +-
.../hadoop/security/token/DtFileOperations.java | 28 +-
.../hadoop/security/token/DtUtilShell.java | 37 +-
.../io/compress/zstd/ZStandardCompressor.c | 11 +-
.../io/compress/zstd/ZStandardDecompressor.c | 1 +
.../src/main/resources/core-default.xml | 8 +
.../src/site/markdown/CommandsManual.md | 1 +
.../src/site/markdown/CredentialProviderAPI.md | 130 +-
.../site/markdown/registry/hadoop-registry.md | 1018 ++++++++++
.../src/site/markdown/registry/index.md | 31 +
.../markdown/registry/registry-configuration.md | 397 ++++
.../src/site/markdown/registry/registry-dns.md | 224 +++
.../site/markdown/registry/registry-security.md | 120 ++
.../using-the-hadoop-service-registry.md | 273 +++
.../hadoop/crypto/key/TestKeyProvider.java | 32 +-
.../TestZStandardCompressorDecompressor.java | 10 +-
.../java/org/apache/hadoop/ipc/TestIPC.java | 53 +-
.../apache/hadoop/security/TestCredentials.java | 57 +-
.../hadoop/security/TestUGILoginFromKeytab.java | 56 +
.../security/TestUserGroupInformation.java | 2 +-
.../hadoop/security/ssl/KeyStoreTestUtil.java | 105 +
.../hadoop/security/token/TestDtUtilShell.java | 44 +
.../dev-support/findbugs-exclude.xml | 33 +
hadoop-common-project/hadoop-registry/pom.xml | 309 +++
.../apache/hadoop/registry/cli/RegistryCli.java | 497 +++++
.../hadoop/registry/client/api/BindFlags.java | 41 +
.../registry/client/api/DNSOperations.java | 60 +
.../client/api/DNSOperationsFactory.java | 78 +
.../registry/client/api/RegistryConstants.java | 388 ++++
.../registry/client/api/RegistryOperations.java | 182 ++
.../client/api/RegistryOperationsFactory.java | 160 ++
.../registry/client/api/package-info.java | 35 +
.../registry/client/binding/JsonSerDeser.java | 117 ++
.../client/binding/RegistryPathUtils.java | 238 +++
.../client/binding/RegistryTypeUtils.java | 291 +++
.../registry/client/binding/RegistryUtils.java | 399 ++++
.../registry/client/binding/package-info.java | 22 +
.../AuthenticationFailedException.java | 39 +
.../exceptions/InvalidPathnameException.java | 40 +
.../exceptions/InvalidRecordException.java | 41 +
.../NoChildrenForEphemeralsException.java | 48 +
.../exceptions/NoPathPermissionsException.java | 45 +
.../client/exceptions/NoRecordException.java | 45 +
.../client/exceptions/RegistryIOException.java | 58 +
.../client/exceptions/package-info.java | 33 +
.../impl/FSRegistryOperationsService.java | 248 +++
.../client/impl/RegistryOperationsClient.java | 55 +
.../registry/client/impl/package-info.java | 26 +
.../client/impl/zk/BindingInformation.java | 41 +
.../registry/client/impl/zk/CuratorService.java | 896 +++++++++
.../registry/client/impl/zk/ListenerHandle.java | 25 +
.../registry/client/impl/zk/PathListener.java | 30 +
.../client/impl/zk/RegistryBindingSource.java | 36 +
.../impl/zk/RegistryInternalConstants.java | 81 +
.../impl/zk/RegistryOperationsService.java | 165 ++
.../client/impl/zk/RegistrySecurity.java | 1143 +++++++++++
.../registry/client/impl/zk/ZKPathDumper.java | 133 ++
.../client/impl/zk/ZookeeperConfigOptions.java | 118 ++
.../registry/client/impl/zk/package-info.java | 39 +
.../registry/client/types/AddressTypes.java | 94 +
.../hadoop/registry/client/types/Endpoint.java | 247 +++
.../registry/client/types/ProtocolTypes.java | 99 +
.../client/types/RegistryPathStatus.java | 123 ++
.../registry/client/types/ServiceRecord.java | 312 +++
.../registry/client/types/package-info.java | 41 +
.../client/types/yarn/PersistencePolicies.java | 50 +
.../types/yarn/YarnRegistryAttributes.java | 42 +
.../registry/conf/RegistryConfiguration.java | 41 +
.../hadoop/registry/conf/package-info.java | 22 +
.../dns/ApplicationServiceRecordProcessor.java | 366 ++++
.../server/dns/BaseServiceRecordProcessor.java | 470 +++++
.../dns/ContainerServiceRecordProcessor.java | 284 +++
.../hadoop/registry/server/dns/LookupTask.java | 39 +
.../dns/PrivilegedRegistryDNSStarter.java | 81 +
.../server/dns/RecordCreatorFactory.java | 275 +++
.../hadoop/registry/server/dns/RegistryDNS.java | 1800 ++++++++++++++++++
.../registry/server/dns/RegistryDNSServer.java | 267 +++
.../registry/server/dns/ReverseZoneUtils.java | 171 ++
.../registry/server/dns/SecureableZone.java | 151 ++
.../server/dns/ServiceRecordProcessor.java | 53 +
.../registry/server/dns/ZoneSelector.java | 33 +
.../registry/server/dns/package-info.java | 26 +
.../integration/SelectByYarnPersistence.java | 60 +
.../server/integration/package-info.java | 23 +
.../hadoop/registry/server/package-info.java | 27 +
.../server/services/AddingCompositeService.java | 56 +
.../services/DeleteCompletionCallback.java | 57 +
.../server/services/MicroZookeeperService.java | 282 +++
.../services/MicroZookeeperServiceKeys.java | 69 +
.../server/services/RegistryAdminService.java | 529 +++++
.../registry/server/services/package-info.java | 39 +
.../src/main/tla/hadoopregistry.tla | 582 ++++++
.../hadoop/registry/AbstractRegistryTest.java | 118 ++
.../hadoop/registry/AbstractZKRegistryTest.java | 113 ++
.../hadoop/registry/RegistryTestHelper.java | 353 ++++
.../hadoop/registry/cli/TestRegistryCli.java | 197 ++
.../client/binding/TestMarshalling.java | 143 ++
.../binding/TestRegistryOperationUtils.java | 56 +
.../client/binding/TestRegistryPathUtils.java | 186 ++
.../client/impl/CuratorEventCatcher.java | 68 +
.../client/impl/TestCuratorService.java | 249 +++
.../impl/TestFSRegistryOperationsService.java | 298 +++
.../client/impl/TestMicroZookeeperService.java | 61 +
.../integration/TestYarnPolicySelector.java | 65 +
.../operations/TestRegistryOperations.java | 331 ++++
.../secure/AbstractSecureRegistryTest.java | 369 ++++
.../registry/secure/KerberosConfiguration.java | 100 +
.../secure/TestRegistrySecurityHelper.java | 211 ++
.../registry/secure/TestSecureLogins.java | 234 +++
.../registry/secure/TestSecureRegistry.java | 213 +++
.../registry/server/dns/TestRegistryDNS.java | 725 +++++++
.../server/dns/TestReverseZoneUtils.java | 89 +
.../server/dns/TestSecureRegistryDNS.java | 44 +
.../test/resources/0.17.172.in-addr.arpa.zone | 36 +
.../src/test/resources/log4j.properties | 63 +
.../src/test/resources/test.private | 32 +
hadoop-common-project/pom.xml | 1 +
.../hadoop/hdds/scm/XceiverClientGrpc.java | 197 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 20 +-
.../hadoop/hdds/scm/XceiverClientRatis.java | 37 +-
.../scm/client/ContainerOperationClient.java | 46 +-
.../hdds/scm/storage/ChunkOutputStream.java | 17 +-
.../common/src/main/conf/log4j.properties | 157 --
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 9 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 125 +-
.../org/apache/hadoop/hdds/cli/GenericCli.java | 5 +-
.../hadoop/hdds/cli/GenericParentCommand.java | 4 +
.../org/apache/hadoop/hdds/client/BlockID.java | 85 +-
.../hadoop/hdds/client/ContainerBlockID.java | 79 +
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 15 +
.../hadoop/hdds/scm/XceiverClientSpi.java | 2 +-
.../hadoop/hdds/scm/client/ScmClient.java | 8 +-
.../hdds/scm/container/ContainerInfo.java | 10 +-
.../common/helpers/AllocatedBlock.java | 22 +-
.../common/helpers/ContainerWithPipeline.java | 3 +-
.../scm/container/common/helpers/Pipeline.java | 315 ---
.../container/common/helpers/PipelineID.java | 97 -
.../hadoop/hdds/scm/pipeline/Pipeline.java | 41 +-
.../scm/pipeline/PipelineNotFoundException.java | 46 +
.../StorageContainerLocationProtocol.java | 2 +-
...kLocationProtocolClientSideTranslatorPB.java | 9 +-
...rLocationProtocolClientSideTranslatorPB.java | 6 +-
.../scm/storage/ContainerProtocolCalls.java | 30 +-
.../hdds/security/x509/SecurityConfig.java | 2 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 16 +-
.../apache/hadoop/ozone/common/BlockGroup.java | 3 +-
.../container/common/helpers/BlockData.java | 8 +-
...kLocationProtocolServerSideTranslatorPB.java | 2 +-
.../org/apache/hadoop/utils/RocksDBStore.java | 4 +-
.../org/apache/hadoop/utils/db/RDBStore.java | 4 +-
.../main/java/org/apache/ratis/RatisHelper.java | 15 +-
.../main/proto/DatanodeContainerProtocol.proto | 34 +-
.../main/proto/ScmBlockLocationProtocol.proto | 2 +-
.../StorageContainerLocationProtocol.proto | 4 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 11 +-
.../common/src/main/resources/ozone-default.xml | 58 +-
.../TestCertificateSignRequest.java | 2 +-
.../x509/certificates/TestRootCertificate.java | 2 +-
.../x509/keys/TestHDDSKeyGenerator.java | 2 +-
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 53 +-
.../container/common/impl/ContainerData.java | 22 +-
.../common/impl/ContainerDataYaml.java | 6 +-
.../container/common/impl/HddsDispatcher.java | 13 +-
.../container/common/interfaces/Container.java | 9 +-
.../statemachine/DatanodeStateMachine.java | 4 +-
.../states/endpoint/VersionEndpointTask.java | 79 +-
.../transport/server/XceiverServerGrpc.java | 2 +-
.../transport/server/ratis/CSMMetrics.java | 5 +-
.../server/ratis/ContainerStateMachine.java | 216 ++-
.../server/ratis/XceiverServerRatis.java | 58 +-
.../container/common/volume/VolumeInfo.java | 19 +-
.../container/common/volume/VolumeSet.java | 11 +-
.../container/keyvalue/KeyValueContainer.java | 31 +-
.../keyvalue/KeyValueContainerData.java | 9 +-
.../container/keyvalue/KeyValueHandler.java | 28 +-
.../container/keyvalue/helpers/BlockUtils.java | 4 +-
.../container/keyvalue/helpers/ChunkUtils.java | 30 +-
.../keyvalue/helpers/SmallFileUtils.java | 7 +
.../keyvalue/impl/BlockManagerImpl.java | 38 +-
.../keyvalue/impl/ChunkManagerImpl.java | 5 +-
.../keyvalue/interfaces/BlockManager.java | 3 +-
.../keyvalue/interfaces/ChunkManager.java | 3 +-
.../container/ozoneimpl/OzoneContainer.java | 5 +
.../replication/ReplicationSupervisor.java | 137 +-
.../commands/CloseContainerCommand.java | 2 +-
.../StorageContainerDatanodeProtocol.proto | 57 +-
.../ozone/container/common/SCMTestUtils.java | 3 +-
.../ozone/container/common/ScmTestMock.java | 14 +-
.../common/TestDatanodeStateMachine.java | 3 +-
.../common/TestKeyValueContainerData.java | 6 +-
.../common/impl/TestContainerDataYaml.java | 8 +-
.../container/common/impl/TestContainerSet.java | 16 +-
.../container/common/volume/TestHddsVolume.java | 9 +-
.../container/common/volume/TestVolumeSet.java | 4 +-
.../keyvalue/TestChunkManagerImpl.java | 37 +-
.../keyvalue/TestKeyValueContainer.java | 15 +-
.../container/keyvalue/TestKeyValueHandler.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 4 +-
.../replication/TestReplicationSupervisor.java | 31 +-
.../hadoop/hdds/server/BaseHttpServer.java | 8 +-
.../apache/hadoop/hdds/server/ServerUtils.java | 49 +-
hadoop-hdds/pom.xml | 161 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 9 +-
.../block/DatanodeDeletedBlockTransactions.java | 4 +-
.../hdds/scm/block/DeletedBlockLogImpl.java | 15 +-
.../container/CloseContainerEventHandler.java | 4 +-
.../hdds/scm/container/ContainerManager.java | 6 +-
.../scm/container/ContainerReportHandler.java | 2 +-
.../scm/container/ContainerStateManager.java | 25 +-
.../hdds/scm/container/SCMContainerManager.java | 92 +-
.../hadoop/hdds/scm/events/SCMEvents.java | 9 -
.../hadoop/hdds/scm/node/NodeManager.java | 4 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 4 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 4 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 30 +-
.../hdds/scm/node/states/Node2PipelineMap.java | 8 +-
.../hdds/scm/pipeline/PipelineFactory.java | 6 +-
.../hdds/scm/pipeline/PipelineManager.java | 10 +-
.../hdds/scm/pipeline/PipelineProvider.java | 2 +-
.../scm/pipeline/PipelineReportHandler.java | 16 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 27 +-
.../hdds/scm/pipeline/PipelineStateMap.java | 91 +-
.../scm/pipeline/RatisPipelineProvider.java | 15 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 43 +-
.../scm/pipeline/SimplePipelineProvider.java | 13 +-
.../pipelines/PipelineActionEventHandler.java | 62 -
.../scm/pipelines/PipelineCloseHandler.java | 52 -
.../hdds/scm/pipelines/PipelineManager.java | 171 --
.../scm/pipelines/PipelineReportHandler.java | 59 -
.../hdds/scm/pipelines/PipelineSelector.java | 481 -----
.../scm/pipelines/PipelineStateManager.java | 136 --
.../hadoop/hdds/scm/pipelines/package-info.java | 38 -
.../scm/pipelines/ratis/RatisManagerImpl.java | 129 --
.../hdds/scm/pipelines/ratis/package-info.java | 18 -
.../standalone/StandaloneManagerImpl.java | 122 --
.../scm/pipelines/standalone/package-info.java | 18 -
.../scm/server/SCMClientProtocolServer.java | 3 +-
.../hadoop/hdds/scm/server/SCMStorage.java | 4 +-
.../scm/server/StorageContainerManager.java | 48 +-
.../apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +-
.../hadoop/hdds/scm/HddsWhiteboxTestUtils.java | 103 +
.../hadoop/hdds/scm/TestHddsServerUtils.java | 203 ++
.../org/apache/hadoop/hdds/scm/TestUtils.java | 20 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 13 +-
.../hdds/scm/block/TestDeletedBlockLog.java | 36 +-
.../hdds/scm/container/MockNodeManager.java | 4 +-
.../TestCloseContainerEventHandler.java | 24 +-
.../container/TestContainerReportHandler.java | 17 +-
.../container/TestContainerStateManager.java | 29 +-
.../scm/container/TestSCMContainerManager.java | 50 +-
.../replication/TestReplicationManager.java | 29 +-
.../hdds/scm/node/TestContainerPlacement.java | 12 +-
.../hdds/scm/node/TestDeadNodeHandler.java | 11 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 6 +-
.../ozone/container/common/TestEndPoint.java | 5 +-
.../testutils/ReplicationNodeManagerMock.java | 4 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 13 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 12 +
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 85 +-
.../hdfs/client/HdfsClientConfigKeys.java | 5 +
.../hdfs/protocol/SnapshotDiffReport.java | 65 +
.../http/server/HttpFSParametersProvider.java | 10 +-
.../fs/http/client/BaseTestHttpFSWith.java | 60 +
.../hadoop/fs/http/server/TestHttpFSServer.java | 65 +-
.../http/server/TestHttpFSServerWebServer.java | 11 +
.../org/apache/hadoop/test/TestHdfsHelper.java | 24 +-
.../src/CMakeLists.txt | 22 +-
.../src/main/native/libhdfspp/CMakeLists.txt | 4 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 12 +-
.../hadoop/hdfs/protocol/BlockListAsLongs.java | 2 +-
.../QJournalProtocolServerSideTranslatorPB.java | 2 +-
.../block/BlockPoolTokenSecretManager.java | 34 +-
.../token/block/BlockTokenSecretManager.java | 39 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 15 +-
.../server/blockmanagement/BlockManager.java | 39 +-
.../blockmanagement/BlockPlacementPolicy.java | 1 -
.../CombinedHostFileManager.java | 6 +-
.../blockmanagement/CorruptReplicasMap.java | 2 +-
.../blockmanagement/DatanodeAdminManager.java | 8 +-
.../blockmanagement/DatanodeDescriptor.java | 4 +-
.../blockmanagement/HeartbeatManager.java | 2 +-
.../server/blockmanagement/HostFileManager.java | 7 +-
.../hdfs/server/blockmanagement/HostSet.java | 8 +-
.../server/blockmanagement/SlowPeerTracker.java | 5 +-
.../server/datanode/BlockPoolSliceStorage.java | 60 +-
.../server/datanode/BlockRecoveryWorker.java | 15 +-
.../hdfs/server/datanode/BlockScanner.java | 6 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 10 +-
.../hdfs/server/datanode/DataStorage.java | 4 +-
.../hdfs/server/datanode/DirectoryScanner.java | 1 -
.../hdfs/server/datanode/FileIoProvider.java | 3 -
.../hdfs/server/datanode/VolumeScanner.java | 4 +-
.../server/datanode/checker/AbstractFuture.java | 13 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 12 +-
.../server/datanode/fsdataset/FsVolumeSpi.java | 13 +-
.../datanode/metrics/OutlierDetector.java | 3 +-
.../diskbalancer/DiskBalancerException.java | 1 -
.../datamodel/DiskBalancerCluster.java | 11 +-
.../datamodel/DiskBalancerDataNode.java | 10 +-
.../diskbalancer/planner/GreedyPlanner.java | 2 +-
.../hadoop/hdfs/server/namenode/AclStorage.java | 18 +-
.../server/namenode/EncryptionZoneManager.java | 42 +-
.../hdfs/server/namenode/FSDirectory.java | 8 +-
.../hdfs/server/namenode/FSNamesystem.java | 44 +-
.../hadoop/hdfs/server/namenode/INode.java | 4 +-
.../hdfs/server/namenode/INodeReference.java | 6 +-
.../hdfs/server/namenode/INodesInPath.java | 4 +-
.../hdfs/server/namenode/JournalManager.java | 2 +-
.../hdfs/server/namenode/LeaseManager.java | 2 +-
.../server/namenode/MetaRecoveryContext.java | 2 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 36 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 16 +-
.../hdfs/server/namenode/NamenodeFsck.java | 9 +-
.../hadoop/hdfs/server/namenode/Quota.java | 5 +-
.../server/namenode/ReencryptionHandler.java | 2 +-
.../server/namenode/XAttrPermissionFilter.java | 4 +-
.../hdfs/server/namenode/XAttrStorage.java | 8 +-
.../snapshot/AbstractINodeDiffList.java | 8 +-
.../namenode/snapshot/DiffListBySkipList.java | 9 +-
.../snapshot/DirectorySnapshottableFeature.java | 5 +
.../namenode/snapshot/SnapshotDiffInfo.java | 50 +-
.../sps/BlockStorageMovementNeeded.java | 5 +-
.../namenode/sps/DatanodeCacheManager.java | 2 +-
.../sps/StoragePolicySatisfyManager.java | 14 +-
.../startupprogress/StartupProgressView.java | 4 +-
.../server/namenode/top/metrics/TopMetrics.java | 17 +-
.../namenode/top/window/RollingWindow.java | 18 +-
.../top/window/RollingWindowManager.java | 2 +-
.../protocol/BlockStorageMovementCommand.java | 11 +-
.../hdfs/server/protocol/DatanodeProtocol.java | 2 +-
.../hdfs/server/protocol/NamenodeProtocol.java | 5 +-
.../sps/ExternalSPSBlockMoveTaskHandler.java | 2 +
.../org/apache/hadoop/hdfs/tools/DFSck.java | 13 +-
.../offlineEditsViewer/OfflineEditsViewer.java | 4 +-
.../offlineEditsViewer/OfflineEditsVisitor.java | 2 +-
.../StatisticsEditsVisitor.java | 4 +-
.../NameDistributionVisitor.java | 4 +-
.../OfflineImageViewerPB.java | 31 +-
.../offlineImageViewer/PBImageTextWriter.java | 4 +-
.../java/org/apache/hadoop/hdfs/util/Diff.java | 16 +-
.../org/apache/hadoop/hdfs/util/XMLUtils.java | 4 +-
.../src/main/resources/hdfs-default.xml | 11 +
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 51 +
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 36 +-
.../apache/hadoop/hdfs/TestHAAuxiliaryPort.java | 112 ++
.../apache/hadoop/hdfs/TestLeaseRecovery2.java | 2 +-
.../security/token/block/TestBlockToken.java | 13 +-
.../TestPendingReconstruction.java | 2 +
...constructStripedBlocksWithRackAwareness.java | 11 +-
.../server/datanode/TestDataNodeLifeline.java | 5 +
.../fsdataset/impl/TestLazyPersistFiles.java | 15 +-
.../TestUpgradeDomainBlockPlacementPolicy.java | 22 +-
.../TestOfflineImageViewer.java | 101 +-
.../hadoop-mapreduce-client-app/pom.xml | 5 +
.../v2/app/client/MRClientService.java | 15 +-
.../v2/app/rm/RMContainerAllocator.java | 14 +-
.../v2/app/rm/TestRMContainerAllocator.java | 14 +
.../mapreduce/v2/app/webapp/TestAMWebApp.java | 158 +-
.../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 6 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 22 +
.../src/main/resources/mapred-default.xml | 21 +
hadoop-maven-plugins/pom.xml | 4 +
.../ozone/client/io/ChunkGroupInputStream.java | 12 +-
.../ozone/client/io/ChunkGroupOutputStream.java | 31 +-
.../ozone/client/TestHddsClientUtils.java | 137 +-
.../src/main/conf/om-audit-log4j2.properties | 90 -
.../common/src/main/conf/ozone-site.xml | 24 -
.../java/org/apache/hadoop/ozone/OmUtils.java | 50 +-
.../apache/hadoop/ozone/om/OMConfigKeys.java | 3 +
.../ozone/om/helpers/OmKeyLocationInfo.java | 19 +-
.../src/main/proto/OzoneManagerProtocol.proto | 1 -
.../org/apache/hadoop/ozone/TestOmUtils.java | 91 +
.../dist/dev-support/bin/dist-layout-stitching | 4 +-
.../dist/src/main/conf/log4j.properties | 157 ++
.../src/main/conf/om-audit-log4j2.properties | 90 +
hadoop-ozone/dist/src/main/conf/ozone-site.xml | 24 +
.../dist/src/main/smoketest/s3/README.md | 2 +-
.../dist/src/main/smoketest/s3/awss3.robot | 4 +-
.../dist/src/main/smoketest/s3/objectcopy.robot | 66 +
.../src/main/smoketest/s3/objectdelete.robot | 6 +-
.../main/smoketest/s3/objectmultidelete.robot | 6 +-
.../src/main/smoketest/s3/objectputget.robot | 2 +-
hadoop-ozone/docs/content/S3.md | 130 ++
.../ozonedoc/layouts/partials/navbar.html | 5 +-
.../ozonedoc/layouts/partials/sidebar.html | 8 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 29 +-
.../hdds/scm/pipeline/TestNodeFailure.java | 27 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 41 +-
.../scm/pipeline/TestPipelineStateManager.java | 171 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 13 +-
.../scm/pipeline/TestSCMPipelineManager.java | 49 +-
.../hdds/scm/pipeline/TestSCMRestart.java | 23 +-
.../pipeline/TestSimplePipelineProvider.java | 13 +-
.../apache/hadoop/ozone/MiniOzoneCluster.java | 16 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 41 +-
.../apache/hadoop/ozone/RatisTestHelper.java | 2 +-
.../TestContainerStateMachineIdempotency.java | 2 +-
.../hadoop/ozone/TestMiniOzoneCluster.java | 82 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 28 +-
.../ozone/TestStorageContainerManager.java | 13 +-
.../TestStorageContainerManagerHelper.java | 5 +-
.../ozone/client/rest/TestOzoneRestClient.java | 4 +-
.../hadoop/ozone/client/rpc/TestBCSID.java | 2 +-
.../rpc/TestCloseContainerHandlingByClient.java | 72 +-
.../rpc/TestContainerStateMachineFailures.java | 23 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 112 +-
.../ozone/container/ContainerTestHelper.java | 76 +-
.../container/TestContainerReplication.java | 6 +-
.../common/impl/TestCloseContainerHandler.java | 10 +-
.../common/impl/TestContainerPersistence.java | 92 +-
.../commandhandler/TestBlockDeletion.java | 14 +-
.../TestCloseContainerByPipeline.java | 8 +-
.../TestCloseContainerHandler.java | 2 +-
.../transport/server/ratis/TestCSMMetrics.java | 14 +-
.../container/metrics/TestContainerMetrics.java | 4 +-
.../container/ozoneimpl/TestOzoneContainer.java | 9 +-
.../ozoneimpl/TestOzoneContainerRatis.java | 6 +-
.../container/server/TestContainerServer.java | 21 +-
.../apache/hadoop/ozone/om/TestOmMetrics.java | 23 +-
.../hadoop/ozone/om/TestOzoneManager.java | 3 +-
.../hadoop/ozone/ozShell/TestOzoneShell.java | 77 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 2 +-
.../ozone/scm/TestContainerSmallFile.java | 51 +
.../TestGetCommittedBlockLengthAndPutKey.java | 18 +-
.../ozone/scm/TestXceiverClientManager.java | 15 +-
.../hadoop/ozone/web/client/TestKeys.java | 6 +-
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 7 +-
.../org/apache/hadoop/ozone/om/OMStorage.java | 5 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 5 +-
.../apache/hadoop/ozone/om/OzoneManager.java | 10 +-
.../hadoop/ozone/web/ozShell/Handler.java | 139 +-
.../hadoop/ozone/web/ozShell/OzoneAddress.java | 251 +++
.../web/ozShell/bucket/BucketCommands.java | 6 +
.../web/ozShell/bucket/CreateBucketHandler.java | 27 +-
.../web/ozShell/bucket/DeleteBucketHandler.java | 20 +-
.../web/ozShell/bucket/InfoBucketHandler.java | 28 +-
.../web/ozShell/bucket/ListBucketHandler.java | 26 +-
.../web/ozShell/bucket/S3BucketMapping.java | 55 +-
.../web/ozShell/bucket/UpdateBucketHandler.java | 20 +-
.../web/ozShell/keys/DeleteKeyHandler.java | 23 +-
.../ozone/web/ozShell/keys/GetKeyHandler.java | 18 +-
.../ozone/web/ozShell/keys/InfoKeyHandler.java | 28 +-
.../ozone/web/ozShell/keys/KeyCommands.java | 6 +
.../ozone/web/ozShell/keys/ListKeyHandler.java | 28 +-
.../ozone/web/ozShell/keys/PutKeyHandler.java | 21 +-
.../web/ozShell/volume/CreateVolumeHandler.java | 28 +-
.../web/ozShell/volume/DeleteVolumeHandler.java | 8 +-
.../web/ozShell/volume/InfoVolumeHandler.java | 8 +-
.../web/ozShell/volume/ListVolumeHandler.java | 27 +-
.../web/ozShell/volume/UpdateVolumeHandler.java | 8 +-
.../web/ozShell/volume/VolumeCommands.java | 6 +
.../ozone/om/ScmBlockLocationTestIngClient.java | 22 +-
.../ozone/web/ozShell/TestOzoneAddress.java | 100 +
.../hadoop/ozone/web/ozShell/package-info.java | 21 +
.../ITestOzoneContractGetFileStatus.java | 6 +-
hadoop-ozone/pom.xml | 139 +-
hadoop-ozone/s3gateway/pom.xml | 8 +-
.../ozone/s3/SignedChunksInputStream.java | 99 +
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 22 +
.../ozone/s3/endpoint/BucketEndpoint.java | 134 +-
.../ozone/s3/endpoint/CopyObjectResponse.java | 63 +
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 49 +-
.../ozone/s3/endpoint/ListObjectResponse.java | 22 +
.../ozone/s3/endpoint/MultiDeleteRequest.java | 2 +-
.../MultiDeleteRequestUnmarshaller.java | 84 +
.../ozone/s3/endpoint/ObjectEndpoint.java | 185 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 6 +-
.../ozone/s3/endpoint/XmlNamespaceFilter.java | 54 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 45 +-
.../s3/header/AuthenticationHeaderParser.java | 61 +
.../ozone/s3/header/AuthorizationHeaderV2.java | 12 +-
.../ozone/s3/header/AuthorizationHeaderV4.java | 12 +-
.../hadoop/ozone/s3/header/Credential.java | 3 +-
.../apache/hadoop/ozone/s3/util/S3Consts.java | 38 +
.../hadoop/ozone/s3/util/S3StorageType.java | 55 +
.../apache/hadoop/ozone/s3/util/S3utils.java | 91 +
.../hadoop/ozone/s3/util/package-info.java | 22 +
.../s3gateway/src/main/resources/browser.html | 4 +-
.../resources/webapps/s3gateway/WEB-INF/web.xml | 4 +
.../resources/webapps/static/images/ozone.ico | Bin 0 -> 1150 bytes
.../hadoop/ozone/client/OzoneBucketStub.java | 7 +-
.../ozone/s3/TestSignedChunksInputStream.java | 84 +
.../ozone/s3/TestVirtualHostStyleFilter.java | 20 +-
.../hadoop/ozone/s3/endpoint/TestBucketGet.java | 227 ++-
.../TestMultiDeleteRequestUnmarshaller.java | 76 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 4 +-
.../s3/endpoint/TestObjectMultiDelete.java | 61 +-
.../hadoop/ozone/s3/endpoint/TestPutObject.java | 164 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 22 +-
.../ozone/s3/exception/TestOS3Exception.java | 2 +-
.../genesis/BenchMarkContainerStateMap.java | 27 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 6 +-
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 2 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 130 ++
.../freon/TestFreonWithDatanodeRestart.java | 103 +
.../apache/hadoop/ozone/om/TestOmSQLCli.java | 3 +-
.../hadoop/ozone/scm/TestContainerSQLCli.java | 10 +-
hadoop-project/pom.xml | 236 +--
.../site/markdown/tools/hadoop-aliyun/index.md | 36 +
.../azurebfs/oauth2/AzureADAuthenticator.java | 7 +-
.../hadoop/tools/mapred/CopyCommitter.java | 8 +-
.../hadoop/tools/mapred/TestCopyCommitter.java | 4 -
hadoop-tools/hadoop-extras/pom.xml | 10 +
hadoop-tools/hadoop-gridmix/pom.xml | 10 +
hadoop-tools/hadoop-streaming/pom.xml | 10 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 1 +
.../hadoop-yarn/conf/yarn-env.sh | 1 +
.../dev-support/findbugs-exclude.xml | 16 -
.../hadoop/yarn/api/ApplicationConstants.java | 9 +-
.../yarn/api/records/ResourceUtilization.java | 8 +-
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop/yarn/service/webapp/ApiServer.java | 7 +-
...RN-Simplified-V1-API-Layer-For-Services.yaml | 154 +-
.../hadoop-yarn-services-core/pom.xml | 2 +-
.../hadoop/yarn/service/ClientAMService.java | 9 +-
.../hadoop/yarn/service/ServiceMaster.java | 7 +
.../hadoop/yarn/service/ServiceScheduler.java | 67 +
.../api/records/ComponentContainers.java | 96 +
.../yarn/service/api/records/ConfigFormat.java | 5 +-
.../yarn/service/client/ServiceClient.java | 147 +-
.../hadoop/yarn/service/utils/ConfigUtils.java | 56 -
.../hadoop/yarn/service/utils/FilterUtils.java | 25 +-
.../hadoop/yarn/service/utils/HttpUtil.java | 123 ++
.../service/utils/PublishedConfiguration.java | 9 -
.../utils/PublishedConfigurationOutputter.java | 2 -
.../yarn/service/utils/ServiceApiUtil.java | 8 +-
.../hadoop/yarn/service/TestServiceAM.java | 32 +-
.../yarn/service/client/TestServiceClient.java | 36 +-
.../yarn/service/utils/TestFilterUtils.java | 39 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 10 +-
.../org/apache/hadoop/yarn/webapp/WebApps.java | 25 +-
.../hadoop/yarn/webapp/view/JQueryUI.java | 6 +-
.../static/dt-1.10.18/css/custom_datatable.css | 68 +
.../webapps/static/dt-1.10.18/css/demo_page.css | 108 ++
.../static/dt-1.10.18/css/demo_table.css | 544 ++++++
.../static/dt-1.10.18/css/jquery.dataTables.css | 466 +++++
.../webapps/static/dt-1.10.18/css/jui-dt.css | 352 ++++
.../static/dt-1.10.18/images/Sorting icons.psd | Bin 0 -> 27490 bytes
.../static/dt-1.10.18/images/back_disabled.jpg | Bin 0 -> 612 bytes
.../static/dt-1.10.18/images/back_enabled.jpg | Bin 0 -> 807 bytes
.../static/dt-1.10.18/images/favicon.ico | Bin 0 -> 894 bytes
.../dt-1.10.18/images/forward_disabled.jpg | Bin 0 -> 635 bytes
.../dt-1.10.18/images/forward_enabled.jpg | Bin 0 -> 852 bytes
.../static/dt-1.10.18/images/sort_asc.png | Bin 0 -> 263 bytes
.../dt-1.10.18/images/sort_asc_disabled.png | Bin 0 -> 252 bytes
.../static/dt-1.10.18/images/sort_both.png | Bin 0 -> 282 bytes
.../static/dt-1.10.18/images/sort_desc.png | Bin 0 -> 260 bytes
.../dt-1.10.18/images/sort_desc_disabled.png | Bin 0 -> 251 bytes
.../dt-1.10.18/js/jquery.dataTables.min.js | 184 ++
.../webapps/static/dt-1.10.7/css/demo_page.css | 110 --
.../webapps/static/dt-1.10.7/css/demo_table.css | 538 ------
.../webapps/static/dt-1.10.7/css/jui-dt.css | 322 ----
.../static/dt-1.10.7/images/Sorting icons.psd | Bin 27490 -> 0 bytes
.../static/dt-1.10.7/images/back_disabled.jpg | Bin 612 -> 0 bytes
.../static/dt-1.10.7/images/back_enabled.jpg | Bin 807 -> 0 bytes
.../webapps/static/dt-1.10.7/images/favicon.ico | Bin 894 -> 0 bytes
.../dt-1.10.7/images/forward_disabled.jpg | Bin 635 -> 0 bytes
.../static/dt-1.10.7/images/forward_enabled.jpg | Bin 852 -> 0 bytes
.../static/dt-1.10.7/images/sort_asc.png | Bin 263 -> 0 bytes
.../dt-1.10.7/images/sort_asc_disabled.png | Bin 252 -> 0 bytes
.../static/dt-1.10.7/images/sort_both.png | Bin 282 -> 0 bytes
.../static/dt-1.10.7/images/sort_desc.png | Bin 260 -> 0 bytes
.../dt-1.10.7/images/sort_desc_disabled.png | Bin 251 -> 0 bytes
.../dt-1.10.7/js/jquery.dataTables.min.js | 160 --
.../dev-support/findbugs-exclude.xml | 21 +
.../hadoop-yarn/hadoop-yarn-csi/pom.xml | 221 +++
.../hadoop/yarn/csi/client/CsiClient.java | 39 +
.../hadoop/yarn/csi/client/CsiClientImpl.java | 51 +
.../hadoop/yarn/csi/client/CsiGrpcClient.java | 126 ++
.../hadoop/yarn/csi/client/package-info.java | 21 +
.../apache/hadoop/yarn/csi/package-info.java | 21 +
.../hadoop/yarn/csi/utils/GrpcHelper.java | 52 +
.../hadoop/yarn/csi/utils/package-info.java | 21 +
.../hadoop-yarn-csi/src/main/proto/csi.proto | 1114 +++++++++++
.../hadoop/yarn/csi/client/FakeCsiDriver.java | 65 +
.../yarn/csi/client/FakeCsiIdentityService.java | 42 +
.../hadoop/yarn/csi/client/TestCsiClient.java | 77 +
.../hadoop/yarn/csi/client/package-info.java | 21 +
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 235 +--
.../apache/hadoop/registry/cli/RegistryCli.java | 497 -----
.../hadoop/registry/client/api/BindFlags.java | 41 -
.../registry/client/api/DNSOperations.java | 60 -
.../client/api/DNSOperationsFactory.java | 78 -
.../registry/client/api/RegistryConstants.java | 390 ----
.../registry/client/api/RegistryOperations.java | 182 --
.../client/api/RegistryOperationsFactory.java | 160 --
.../registry/client/api/package-info.java | 35 -
.../registry/client/binding/JsonSerDeser.java | 117 --
.../client/binding/RegistryPathUtils.java | 238 ---
.../client/binding/RegistryTypeUtils.java | 291 ---
.../registry/client/binding/RegistryUtils.java | 399 ----
.../registry/client/binding/package-info.java | 22 -
.../AuthenticationFailedException.java | 39 -
.../exceptions/InvalidPathnameException.java | 40 -
.../exceptions/InvalidRecordException.java | 41 -
.../NoChildrenForEphemeralsException.java | 48 -
.../exceptions/NoPathPermissionsException.java | 45 -
.../client/exceptions/NoRecordException.java | 45 -
.../client/exceptions/RegistryIOException.java | 58 -
.../client/exceptions/package-info.java | 33 -
.../impl/FSRegistryOperationsService.java | 249 ---
.../client/impl/RegistryOperationsClient.java | 55 -
.../registry/client/impl/package-info.java | 26 -
.../client/impl/zk/BindingInformation.java | 41 -
.../registry/client/impl/zk/CuratorService.java | 896 ---------
.../registry/client/impl/zk/ListenerHandle.java | 25 -
.../registry/client/impl/zk/PathListener.java | 30 -
.../client/impl/zk/RegistryBindingSource.java | 36 -
.../impl/zk/RegistryInternalConstants.java | 81 -
.../impl/zk/RegistryOperationsService.java | 165 --
.../client/impl/zk/RegistrySecurity.java | 1143 -----------
.../registry/client/impl/zk/ZKPathDumper.java | 133 --
.../client/impl/zk/ZookeeperConfigOptions.java | 118 --
.../registry/client/impl/zk/package-info.java | 39 -
.../registry/client/types/AddressTypes.java | 94 -
.../hadoop/registry/client/types/Endpoint.java | 247 ---
.../registry/client/types/ProtocolTypes.java | 99 -
.../client/types/RegistryPathStatus.java | 123 --
.../registry/client/types/ServiceRecord.java | 312 ---
.../registry/client/types/package-info.java | 41 -
.../client/types/yarn/PersistencePolicies.java | 50 -
.../types/yarn/YarnRegistryAttributes.java | 42 -
.../dns/ApplicationServiceRecordProcessor.java | 366 ----
.../server/dns/BaseServiceRecordProcessor.java | 470 -----
.../dns/ContainerServiceRecordProcessor.java | 284 ---
.../hadoop/registry/server/dns/LookupTask.java | 39 -
.../dns/PrivilegedRegistryDNSStarter.java | 80 -
.../server/dns/RecordCreatorFactory.java | 275 ---
.../hadoop/registry/server/dns/RegistryDNS.java | 1800 ------------------
.../registry/server/dns/RegistryDNSServer.java | 267 ---
.../registry/server/dns/ReverseZoneUtils.java | 171 --
.../registry/server/dns/SecureableZone.java | 151 --
.../server/dns/ServiceRecordProcessor.java | 53 -
.../registry/server/dns/ZoneSelector.java | 33 -
.../registry/server/dns/package-info.java | 26 -
.../integration/SelectByYarnPersistence.java | 60 -
.../server/integration/package-info.java | 23 -
.../hadoop/registry/server/package-info.java | 27 -
.../server/services/AddingCompositeService.java | 56 -
.../services/DeleteCompletionCallback.java | 57 -
.../server/services/MicroZookeeperService.java | 282 ---
.../services/MicroZookeeperServiceKeys.java | 69 -
.../server/services/RegistryAdminService.java | 529 -----
.../registry/server/services/package-info.java | 39 -
.../src/main/resources/.keep | 0
.../src/main/tla/yarnregistry.tla | 582 ------
.../hadoop/registry/AbstractRegistryTest.java | 118 --
.../hadoop/registry/AbstractZKRegistryTest.java | 113 --
.../hadoop/registry/RegistryTestHelper.java | 353 ----
.../hadoop/registry/cli/TestRegistryCli.java | 197 --
.../client/binding/TestMarshalling.java | 143 --
.../binding/TestRegistryOperationUtils.java | 56 -
.../client/binding/TestRegistryPathUtils.java | 186 --
.../client/impl/CuratorEventCatcher.java | 68 -
.../client/impl/TestCuratorService.java | 249 ---
.../impl/TestFSRegistryOperationsService.java | 298 ---
.../client/impl/TestMicroZookeeperService.java | 60 -
.../integration/TestYarnPolicySelector.java | 65 -
.../operations/TestRegistryOperations.java | 331 ----
.../secure/AbstractSecureRegistryTest.java | 368 ----
.../registry/secure/KerberosConfiguration.java | 100 -
.../secure/TestRegistrySecurityHelper.java | 211 --
.../registry/secure/TestSecureLogins.java | 231 ---
.../registry/secure/TestSecureRegistry.java | 213 ---
.../registry/server/dns/TestRegistryDNS.java | 725 -------
.../server/dns/TestReverseZoneUtils.java | 89 -
.../server/dns/TestSecureRegistryDNS.java | 44 -
.../test/resources/0.17.172.in-addr.arpa.zone | 36 -
.../src/test/resources/log4j.properties | 63 -
.../src/test/resources/test.private | 32 -
.../hadoop-yarn-server-common/pom.xml | 2 +-
.../hadoop/yarn/server/AMRMClientRelayer.java | 55 +-
.../policies/router/LoadBasedRouterPolicy.java | 6 +-
.../yarn/server/uam/UnmanagedAMPoolManager.java | 28 +
.../server/uam/UnmanagedApplicationManager.java | 28 +-
.../yarn/server/MockResourceManagerFacade.java | 5 +-
.../yarn/server/TestAMRMClientRelayer.java | 10 +-
.../router/TestLoadBasedRouterPolicy.java | 31 +
.../metrics/TestAMRMClientRelayerMetrics.java | 6 -
.../uam/TestUnmanagedApplicationManager.java | 27 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 2 +-
.../server/nodemanager/ContainerExecutor.java | 12 +
.../nodemanager/DefaultContainerExecutor.java | 7 +
.../nodemanager/LinuxContainerExecutor.java | 43 +
.../amrmproxy/FederationInterceptor.java | 18 +-
.../linux/privileged/PrivilegedOperation.java | 6 +-
.../linux/resources/CGroupsHandler.java | 2 +-
.../linux/resources/CGroupsHandlerImpl.java | 4 +-
.../CGroupsMemoryResourceHandlerImpl.java | 25 -
.../linux/resources/DefaultOOMHandler.java | 45 +-
.../linux/resources/MemoryResourceHandler.java | 10 -
.../runtime/DockerLinuxContainerRuntime.java | 18 +
.../linux/runtime/docker/DockerRunCommand.java | 6 +
.../monitor/ContainersMonitor.java | 6 +-
.../monitor/ContainersMonitorImpl.java | 119 +-
...locationBasedResourceUtilizationTracker.java | 27 +-
.../scheduler/ContainerScheduler.java | 5 +-
.../nodemanager/webapp/NMWebServices.java | 27 +
.../impl/container-executor.c | 127 +-
.../impl/container-executor.h | 22 +-
.../main/native/container-executor/impl/main.c | 30 +-
.../impl/utils/string-utils.c | 9 +
.../impl/utils/string-utils.h | 6 +
.../test/test-container-executor.c | 115 +-
.../nodemanager/TestLinuxContainerExecutor.java | 11 +
.../amrmproxy/TestAMRMProxyService.java | 1 -
.../TestContainerManagerRecovery.java | 2 +-
.../linux/resources/TestCGroupsHandlerImpl.java | 2 +-
.../TestCGroupsMemoryResourceHandlerImpl.java | 44 -
.../linux/resources/TestDefaultOOMHandler.java | 434 ++++-
.../TestContainersMonitorResourceChange.java | 4 +
...locationBasedResourceUtilizationTracker.java | 18 -
.../TestContainerSchedulerRecovery.java | 79 +-
.../server/resourcemanager/ClientRMService.java | 12 +-
.../server/resourcemanager/RMAuditLogger.java | 81 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 19 +-
.../scheduler/fair/FSAppAttempt.java | 5 +-
.../scheduler/fifo/FifoAppAttempt.java | 10 +-
.../webapp/dao/ReservationListInfo.java | 4 +
.../resourcemanager/TestRMAuditLogger.java | 23 +-
.../capacity/TestContainerAllocation.java | 4 +
.../security/TestRMDelegationTokens.java | 12 +-
.../hadoop/yarn/server/router/Router.java | 8 +
.../hadoop/yarn/server/router/TestRouter.java | 38 +
.../pom.xml | 8 +
.../pom.xml | 8 +
.../collector/TimelineCollectorManager.java | 2 +-
.../src/site/markdown/CapacityScheduler.md | 2 +-
.../src/site/markdown/DockerContainers.md | 16 +
.../src/site/markdown/NodeManager.md | 10 +-
.../site/markdown/NodeManagerCGroupsMemory.md | 12 +-
.../src/site/markdown/NodeManagerCgroups.md | 4 +-
.../src/site/markdown/TimelineServiceV2.md | 17 +
.../src/site/markdown/registry/index.md | 30 -
.../markdown/registry/registry-configuration.md | 397 ----
.../site/markdown/registry/registry-security.md | 120 --
.../registry/using-the-yarn-service-registry.md | 273 ---
.../src/site/markdown/registry/yarn-registry.md | 1018 ----------
.../src/site/markdown/yarn-service/Overview.md | 2 +-
.../site/markdown/yarn-service/RegistryDNS.md | 200 --
.../markdown/yarn-service/ServiceDiscovery.md | 83 +-
.../markdown/yarn-service/ServiceUpgrade.md | 38 +-
hadoop-yarn-project/hadoop-yarn/pom.xml | 1 +
pom.xml | 2 -
758 files changed, 33956 insertions(+), 25918 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 9fd5c63,abacafe..b41baa9
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@@ -97,38 -97,11 +97,43 @@@ public final class HddsConfigKeys
"hdds.lock.max.concurrency";
public static final int HDDS_LOCK_MAX_CONCURRENCY_DEFAULT = 100;
+ public static final String HDDS_KEY_LEN = "hdds.key.len";
+ public static final int HDDS_DEFAULT_KEY_LEN = 2048;
+ public static final String HDDS_KEY_ALGORITHM = "hdds.key.algo";
+ public static final String HDDS_DEFAULT_KEY_ALGORITHM = "RSA";
+ public static final String HDDS_SECURITY_PROVIDER = "hdds.security.provider";
+ public static final String HDDS_DEFAULT_SECURITY_PROVIDER = "BC";
+ public static final String HDDS_KEY_DIR_NAME = "hdds.key.dir.name";
+ public static final String HDDS_KEY_DIR_NAME_DEFAULT = "keys";
+
+ // TODO : Talk to StorageIO classes and see if they can return a secure
+ // storage location for each node.
+ public static final String HDDS_METADATA_DIR_NAME = "hdds.metadata.dir";
+ public static final String HDDS_PRIVATE_KEY_FILE_NAME =
+ "hdds.priv.key.file.name";
+ public static final String HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT = "private.pem";
+ public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
+ + ".name";
+ public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
+
+ /**
+ * Maximum duration of certificates issued by SCM including Self-Signed Roots.
+ * The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
+ * Default value is 5 years and written as P1865D.
+ */
+ public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
+ // Limit Certificate duration to a max value of 5 years.
+ public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
+
+ public static final String HDDS_X509_SIGNATURE_ALGO =
+ "hdds.x509.signature.algorithm";
+ public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
+
-
-
+ // This configuration setting is used as a fallback location by all
+ // Ozone/HDDS services for their metadata. It is useful as a single
+ // config point for test/PoC clusters.
+ //
+ // In any real cluster where performance matters, the SCM, OM and DN
+ // metadata locations must be configured explicitly.
+ public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 9231c8a,0000000..9e0231a
mode 100644,000000..100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@@ -1,231 -1,0 +1,231 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.Provider;
+import java.security.Security;
+import java.time.Duration;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_LEN;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO_DEFAULT;
- import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
++import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * A class that deals with all Security related configs in HDDS.
+ *
+ * This class allows security configs to be read and used consistently across
+ * all of security related code base.
+ */
+public class SecurityConfig {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SecurityConfig.class);
+ private static volatile Provider provider;
+ private final Configuration configuration;
+ private final int size;
+ private final String keyAlgo;
+ private final String providerString;
+ private final String metadatDir;
+ private final String keyDir;
+ private final String privateKeyFileName;
+ private final String publicKeyFileName;
+ private final Duration certDuration;
+ private final String x509SignatureAlgo;
+
+ /**
+ * Constructs a SecurityConfig.
+ *
+ * @param configuration - HDDS Configuration
+ */
+ public SecurityConfig(Configuration configuration) {
+ Preconditions.checkNotNull(configuration, "Configuration cannot be null");
+ this.configuration = configuration;
+ this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
+ this.keyAlgo = this.configuration.get(HDDS_KEY_ALGORITHM,
+ HDDS_DEFAULT_KEY_ALGORITHM);
+ this.providerString = this.configuration.get(HDDS_SECURITY_PROVIDER,
+ HDDS_DEFAULT_SECURITY_PROVIDER);
+
+ // Please Note: To make it easy for our customers we will attempt to read
+ // HDDS metadata dir and if that is not set, we will use Ozone directory.
+ // TODO: We might want to fix this later.
+ this.metadatDir = this.configuration.get(HDDS_METADATA_DIR_NAME,
+ configuration.get(OZONE_METADATA_DIRS));
+
+ Preconditions.checkNotNull(this.metadatDir, "Metadata directory can't be"
+ + " null. Please check configs.");
+ this.keyDir = this.configuration.get(HDDS_KEY_DIR_NAME,
+ HDDS_KEY_DIR_NAME_DEFAULT);
+ this.privateKeyFileName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
+ HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT);
+ this.publicKeyFileName = this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
+ HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT);
+
+ String durationString = this.configuration.get(HDDS_X509_MAX_DURATION,
+ HDDS_X509_MAX_DURATION_DEFAULT);
+ this.certDuration = Duration.parse(durationString);
+ this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO,
+ HDDS_X509_SIGNATURE_ALGO_DEFAULT);
+
+ // First Startup -- if the provider is null, check for the provider.
+ if (SecurityConfig.provider == null) {
+ synchronized (SecurityConfig.class) {
+ provider = Security.getProvider(this.providerString);
+ if (SecurityConfig.provider == null) {
+ // Provider not found, let us try to Dynamically initialize the
+ // provider.
+ provider = initSecurityProvider(this.providerString);
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the public key file name, This is used for storing the public
+ * keys on disk.
+ *
+ * @return String, File name used for public keys.
+ */
+ public String getPublicKeyFileName() {
+ return publicKeyFileName;
+ }
+
+ /**
+ * Returns the private key file name.This is used for storing the private
+ * keys on disk.
+ *
+ * @return String, File name used for private keys.
+ */
+ public String getPrivateKeyFileName() {
+ return privateKeyFileName;
+ }
+
+ /**
+ * Returns the File path to where keys are stored.
+ *
+ * @return String Key location.
+ */
+ public Path getKeyLocation() {
+ return Paths.get(metadatDir, keyDir);
+ }
+
+ /**
+ * Gets the Key Size, The default key size is 2048, since the default
+ * algorithm used is RSA. User can change this by setting the "hdds.key
+ * .len" in configuration.
+ *
+ * @return key size.
+ */
+ public int getSize() {
+ return size;
+ }
+
+ /**
+ * Returns the Provider name. SCM defaults to using Bouncy Castle and will
+ * return "BC".
+ *
+ * @return String Provider name.
+ */
+ public String getProvider() {
+ return providerString;
+ }
+
+ /**
+ * Returns the Key generation Algorithm used. User can change this by
+ * setting the "hdds.key.algo" in configuration.
+ *
+ * @return String Algo.
+ */
+ public String getKeyAlgo() {
+ return keyAlgo;
+ }
+
+ /**
+ * Returns the X.509 Signature Algorithm used. This can be changed by setting
+ * "hdds.x509.signature.algorithm" to the new name. The default algorithm
+ * is SHA256withRSA.
+ *
+ * @return String
+ */
+ public String getSignatureAlgo() {
+ return x509SignatureAlgo;
+ }
+
+ /**
+ * Returns the Configuration used for initializing this SecurityConfig.
+ *
+ * @return Configuration
+ */
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ /**
+ * Returns the maximum length a certificate can be valid in SCM. The
+ * default value is 5 years. This can be changed by setting
+ * "hdds.x509.max.duration" in configuration. The formats accepted are
+ * based on the ISO-8601 duration format PnDTnHnMn.nS
+ *
+ * Default value is 5 years and written as P1865D.
+ *
+ * @return Duration.
+ */
+ public Duration getMaxCertificateDuration() {
+ return this.certDuration;
+ }
+
+ /**
+ * Adds a security provider dynamically if it is not loaded already.
+ *
+ * @param providerName - name of the provider.
+ */
+ private Provider initSecurityProvider(String providerName) {
+ switch (providerName) {
+ case "BC":
+ Security.addProvider(new BouncyCastleProvider());
+ return Security.getProvider(providerName);
+ default:
+ LOG.error("Security Provider:{} is unknown", provider);
+ throw new SecurityException("Unknown security provider:" + provider);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index 0b9ef31,0000000..25f84ff
mode 100644,000000..100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@@ -1,285 -1,0 +1,285 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.operator.ContentVerifierProvider;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentVerifierProviderBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCSException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.util.UUID;
+
- import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
++import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+public class TestCertificateSignRequest {
+
+ private SecurityConfig securityConfig;
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void init() throws IOException {
+ conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+ securityConfig = new SecurityConfig(conf);
+ }
+
+ @Test
+ public void testGenerateCSR() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException,
+ OperatorCreationException, PKCSException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+ PKCS10CertificationRequest csr = builder.build();
+
+ // Check the Subject Name is in the expected format.
+ String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+ // Verify the public key info match
+ byte[] encoded = keyPair.getPublic().getEncoded();
+ SubjectPublicKeyInfo subjectPublicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+ SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+ Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+ // Verify CSR with attribute for extensions
+ Assert.assertEquals(1, csr.getAttributes().length);
+ Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+ // Verify basic constraints extension
+ Extension basicExt = extensions.getExtension(Extension
+ .basicConstraints);
+ Assert.assertEquals(true, basicExt.isCritical());
+
+ // Verify key usage extension
+ Extension keyUsageExt = extensions.getExtension(Extension.keyUsage);
+ Assert.assertEquals(true, keyUsageExt.isCritical());
+
+
+ // Verify San extension not set
+ Assert.assertEquals(null,
+ extensions.getExtension(Extension.subjectAlternativeName));
+
+ // Verify signature in CSR
+ ContentVerifierProvider verifierProvider =
+ new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+ .getProvider()).build(csr.getSubjectPublicKeyInfo());
+ Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+ }
+
+ @Test
+ public void testGenerateCSRwithSan() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException,
+ OperatorCreationException, PKCSException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+
+ // Multi-home
+ builder.addIpAddress("192.168.1.1");
+ builder.addIpAddress("192.168.2.1");
+
+ builder.addDnsName("dn1.abc.com");
+ builder.addRfc822Name("test@abc.com");
+
+ PKCS10CertificationRequest csr = builder.build();
+
+ // Check the Subject Name is in the expected format.
+ String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+ // Verify the public key info match
+ byte[] encoded = keyPair.getPublic().getEncoded();
+ SubjectPublicKeyInfo subjectPublicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+ SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+ Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+ // Verify CSR with attribute for extensions
+ Assert.assertEquals(1, csr.getAttributes().length);
+ Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+ // Verify key usage extension
+ Extension sanExt = extensions.getExtension(Extension.keyUsage);
+ Assert.assertEquals(true, sanExt.isCritical());
+
+
+ // Verify signature in CSR
+ ContentVerifierProvider verifierProvider =
+ new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+ .getProvider()).build(csr.getSubjectPublicKeyInfo());
+ Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+ }
+
+ @Test
+ public void testGenerateCSRWithInvalidParams() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+
+ try {
+ builder.setKey(null);
+ builder.build();
+ Assert.fail("Null Key should have failed.");
+ } catch (NullPointerException | IllegalArgumentException e) {
+ builder.setKey(keyPair);
+ }
+
+ // Now try with blank/null Subject.
+ try {
+ builder.setSubject(null);
+ builder.build();
+ Assert.fail("Null/Blank Subject should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setSubject(subject);
+ }
+
+ try {
+ builder.setSubject("");
+ builder.build();
+ Assert.fail("Null/Blank Subject should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setSubject(subject);
+ }
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setScmID(null);
+ builder.build();
+ Assert.fail("Null/Blank SCM ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setScmID(scmID);
+ }
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setClusterID(null);
+ builder.build();
+ Assert.fail("Null/Blank Cluster ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setClusterID(clusterID);
+ }
+
+ // Now try with invalid IP address
+ try {
+ builder.addIpAddress("255.255.255.*");
+ builder.build();
+ Assert.fail("Invalid ip address");
+ } catch (IllegalArgumentException e) {
+ }
+
+ PKCS10CertificationRequest csr = builder.build();
+
+ // Check the Subject Name is in the expected format.
+ String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+ // Verify the public key info match
+ byte[] encoded = keyPair.getPublic().getEncoded();
+ SubjectPublicKeyInfo subjectPublicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+ SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+ Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+ // Verify CSR with attribute for extensions
+ Assert.assertEquals(1, csr.getAttributes().length);
+ }
+
+ @Test
+ public void testCsrSerialization() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException, IOException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+ PKCS10CertificationRequest csr = builder.build();
+ byte[] csrBytes = csr.getEncoded();
+
+ // Verify de-serialized CSR matches with the original CSR
+ PKCS10CertificationRequest dsCsr = new PKCS10CertificationRequest(csrBytes);
+ Assert.assertEquals(csr, dsCsr);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
index 3a00ca0,0000000..5f99e2c
mode 100644,000000..100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@@ -1,258 -1,0 +1,258 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.SignatureException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Date;
+import java.util.UUID;
+
- import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
++import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * Test Class for Root Certificate generation.
+ */
+public class TestRootCertificate {
+ private SecurityConfig securityConfig;
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void init() throws IOException {
+ conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+ securityConfig = new SecurityConfig(conf);
+ }
+
+ @Test
+ public void testAllFieldsAreExpected()
+ throws SCMSecurityException, NoSuchProviderException,
+ NoSuchAlgorithmException, CertificateException,
+ SignatureException, InvalidKeyException {
+ Instant now = Instant.now();
+ Date notBefore = Date.from(now);
+ Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "testRootCert";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ SelfSignedCertificate.Builder builder =
+ SelfSignedCertificate.newBuilder()
+ .setBeginDate(notBefore)
+ .setEndDate(notAfter)
+ .setClusterID(clusterID)
+ .setScmID(scmID)
+ .setSubject(subject)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+
+ X509CertificateHolder certificateHolder = builder.build();
+
+ //Assert that we indeed have a self signed certificate.
+ Assert.assertEquals(certificateHolder.getIssuer(),
+ certificateHolder.getSubject());
+
+
+ // Make sure that NotBefore is before the current Date
+ Date invalidDate = Date.from(now.minus(Duration.ofDays(1)));
+ Assert.assertFalse(
+ certificateHolder.getNotBefore()
+ .before(invalidDate));
+
+ //Make sure the end date is honored.
+ invalidDate = Date.from(now.plus(Duration.ofDays(366)));
+ Assert.assertFalse(
+ certificateHolder.getNotAfter()
+ .after(invalidDate));
+
+ // Check the Subject Name and Issuer Name is in the expected format.
+ String dnName = String.format(SelfSignedCertificate.getNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(certificateHolder.getIssuer().toString(), dnName);
+ Assert.assertEquals(certificateHolder.getSubject().toString(), dnName);
+
+ // We did not ask for this Certificate to be a CA certificate, hence that
+ // extension should be null.
+ Assert.assertNull(
+ certificateHolder.getExtension(Extension.basicConstraints));
+
+ // Extract the Certificate and verify that certificate matches the public
+ // key.
+ X509Certificate cert =
+ new JcaX509CertificateConverter().getCertificate(certificateHolder);
+ cert.verify(keyPair.getPublic());
+ }
+
+ @Test
+ public void testCACert()
+ throws SCMSecurityException, NoSuchProviderException,
+ NoSuchAlgorithmException {
+ Instant now = Instant.now();
+ Date notBefore = Date.from(now);
+ Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "testRootCert";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ SelfSignedCertificate.Builder builder =
+ SelfSignedCertificate.newBuilder()
+ .setBeginDate(notBefore)
+ .setEndDate(notAfter)
+ .setClusterID(clusterID)
+ .setScmID(scmID)
+ .setSubject(subject)
+ .setKey(keyPair)
+ .setConfiguration(conf)
+ .makeCA();
+
+ X509CertificateHolder certificateHolder = builder.build();
+ // This time we asked for a CA Certificate, make sure that extension is
+ // present and valid.
+ Extension basicExt =
+ certificateHolder.getExtension(Extension.basicConstraints);
+
+ Assert.assertNotNull(basicExt);
+ Assert.assertTrue(basicExt.isCritical());
+
+ // Since this code assigns ONE for the root certificate, we check if the
+ // serial number is the expected number.
+ Assert.assertEquals(certificateHolder.getSerialNumber(), BigInteger.ONE);
+ }
+
+ @Test
+ public void testInvalidParamFails()
+ throws SCMSecurityException, NoSuchProviderException,
+ NoSuchAlgorithmException {
+ Instant now = Instant.now();
+ Date notBefore = Date.from(now);
+ Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "testRootCert";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ SelfSignedCertificate.Builder builder =
+ SelfSignedCertificate.newBuilder()
+ .setBeginDate(notBefore)
+ .setEndDate(notAfter)
+ .setClusterID(clusterID)
+ .setScmID(scmID)
+ .setSubject(subject)
+ .setConfiguration(conf)
+ .setKey(keyPair)
+ .makeCA();
+ try {
+ builder.setKey(null);
+ builder.build();
+ Assert.fail("Null Key should have failed.");
+ } catch (NullPointerException | IllegalArgumentException e) {
+ builder.setKey(keyPair);
+ }
+
+ // Now try with Blank Subject.
+ try {
+ builder.setSubject("");
+ builder.build();
+ Assert.fail("Null/Blank Subject should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setSubject(subject);
+ }
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setScmID(null);
+ builder.build();
+ Assert.fail("Null/Blank SCM ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setScmID(scmID);
+ }
+
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setClusterID(null);
+ builder.build();
+ Assert.fail("Null/Blank Cluster ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setClusterID(clusterID);
+ }
+
+
+ // Swap the Begin and End Date and verify that we cannot create a
+ // certificate like that.
+ try {
+ builder.setBeginDate(notAfter);
+ builder.setEndDate(notBefore);
+ builder.build();
+ Assert.fail("Illegal dates should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setBeginDate(notBefore);
+ builder.setEndDate(notAfter);
+ }
+
+ try {
+ KeyPair newKey = keyGen.generateKey();
+ KeyPair wrongKey = new KeyPair(newKey.getPublic(), keyPair.getPrivate());
+ builder.setKey(wrongKey);
+ X509CertificateHolder certificateHolder = builder.build();
+ X509Certificate cert =
+ new JcaX509CertificateConverter().getCertificate(certificateHolder);
+ cert.verify(wrongKey.getPublic());
+ Assert.fail("Invalid Key, should have thrown.");
+ } catch (SCMSecurityException | CertificateException
+ | SignatureException | InvalidKeyException e) {
+ builder.setKey(keyPair);
+ }
+ // Assert that we can create a certificate with all sane params.
+ Assert.assertNotNull(builder.build());
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
index f9541a2,0000000..08761f4
mode 100644,000000..100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
@@@ -1,87 -1,0 +1,87 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.keys;
+
- import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
++import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PublicKey;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.PKCS8EncodedKeySpec;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for HDDS Key Generator.
+ */
+public class TestHDDSKeyGenerator {
+ private SecurityConfig config;
+
+ @Before
+ public void init() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_METADATA_DIRS, GenericTestUtils.getTempPath("testpath"));
+ config = new SecurityConfig(conf);
+ }
+ /**
+ * In this test we verify that we are able to create a key pair, then get
+ * bytes of that and use ASN1. parser to parse it back to a private key.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ @Test
+ public void testGenerateKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+ Assert.assertEquals(config.getKeyAlgo(),
+ keyPair.getPrivate().getAlgorithm());
+ PKCS8EncodedKeySpec keySpec =
+ new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
+ Assert.assertEquals("PKCS#8", keySpec.getFormat());
+ }
+
+ /**
+ * In this test we assert that size that we specified is used for Key
+ * generation.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ @Test
+ public void testGenerateKeyWithSize() throws NoSuchProviderException,
+ NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey(4096);
+ PublicKey publicKey = keyPair.getPublic();
+ if(publicKey instanceof RSAPublicKey) {
+ Assert.assertEquals(4096,
+ ((RSAPublicKey)(publicKey)).getModulus().bitLength());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index 072ca49,40b3d7a..2cffa74
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@@ -145,28 -161,6 +161,28 @@@ public final class HddsServerUtil
}
/**
+ * Retrieve the socket address that should be used by scm security server to
+ * service clients.
+ *
+ * @param conf
+ * @return Target InetSocketAddress for the SCM security service.
+ */
+ public static InetSocketAddress getScmSecurityInetAddress(
+ Configuration conf) {
+ final Optional<String> host = getHostNameFromConfigKeys(conf,
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY);
+
+ final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY);
+
+ return NetUtils.createSocketAddr(
- host.or(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
++ host.orElse(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
+ ":" + port
- .or(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
++ .orElse(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT)));
+ }
+
+ /**
* Retrieve the socket address that should be used by DataNodes to connect
* to the SCM.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
index 84131fe,aa940df..e8e2b1a
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
@@@ -32,10 -35,8 +35,9 @@@ import org.apache.hadoop.hdds.scm.serve
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
- import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7119be30/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-8905. [Router] Add
JvmMetricsInfo and pause monitor. Contributed by Bilwa S T.
Posted by ae...@apache.org.
YARN-8905. [Router] Add JvmMetricsInfo and pause monitor. Contributed by Bilwa S T.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f84a278b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f84a278b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f84a278b
Branch: refs/heads/HDDS-4
Commit: f84a278baad6cb871ea4196257f23a938826ca23
Parents: 989715e
Author: bibinchundatt <bi...@apache.org>
Authored: Sat Nov 3 20:35:31 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Sat Nov 3 20:35:31 2018 +0530
----------------------------------------------------------------------
.../hadoop/yarn/server/router/Router.java | 8 +++++
.../hadoop/yarn/server/router/TestRouter.java | 38 ++++++++++++++++++++
2 files changed, 46 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84a278b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
index 76050d0..b55c5d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
@@ -24,7 +24,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.JvmPauseMonitor;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
@@ -64,6 +66,7 @@ public class Router extends CompositeService {
private static CompositeServiceShutdownHook routerShutdownHook;
private Configuration conf;
private AtomicBoolean isStopping = new AtomicBoolean(false);
+ private JvmPauseMonitor pauseMonitor;
private RouterClientRMService clientRMProxyService;
private RouterRMAdminService rmAdminProxyService;
private WebApp webApp;
@@ -100,6 +103,11 @@ public class Router extends CompositeService {
WebAppUtils.getRouterWebAppURLWithoutScheme(this.conf));
// Metrics
DefaultMetricsSystem.initialize(METRICS_NAME);
+ JvmMetrics jm = JvmMetrics.initSingleton("Router", null);
+ pauseMonitor = new JvmPauseMonitor();
+ addService(pauseMonitor);
+ jm.setPauseMonitor(pauseMonitor);
+
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84a278b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
new file mode 100644
index 0000000..bf0c688
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+
+/**
+ * Tests {@link Router}.
+ */
+public class TestRouter {
+
+ @Test
+ public void testJVMMetricsService() {
+ YarnConfiguration conf = new YarnConfiguration();
+ Router router = new Router();
+ router.init(conf);
+ assertEquals(3, router.getServices().size());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-8871. Document ATSv2
integrated LogWebService. Contributed by Suma Shivaprasad.
Posted by ae...@apache.org.
YARN-8871. Document ATSv2 integrated LogWebService. Contributed by Suma Shivaprasad.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a283da21
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a283da21
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a283da21
Branch: refs/heads/HDDS-4
Commit: a283da21670c812a391d4c9ee98ebef22fc93868
Parents: 4ec4ec6
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Oct 30 11:34:20 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Oct 30 11:34:20 2018 +0530
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a283da21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
index 04948ce..2314f30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
@@ -1570,3 +1570,8 @@ With this API, you can query set of available entity types for a given app id. I
1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned.
1. If flow context information cannot be retrieved or entity for the given entity id cannot be found, HTTP 404 (Not Found) is returned.
1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned.
+
+## <a name="Aggregated Log Serving for Historical Apps"></a>Aggregated Log Serving for Historical Apps
+
+ TimelineService v.2 supports serving aggregated logs of historical apps. To enable this, configure "yarn.log.server.web-service.url" to "${yarn .timeline-service.hostname}:8188/ws/v2/applicationlog"
+ in `yarn-site.xml`
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org