You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/03/28 19:36:16 UTC
[48/48] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30473ecf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30473ecf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30473ecf
Branch: refs/heads/HDFS-7240
Commit: 30473ecfb426990fbefc7a37b725c799a25c609a
Parents: 643c5e5 8831b18
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Mar 28 10:34:52 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Mar 28 10:34:52 2016 -0700
----------------------------------------------------------------------
BUILDING.txt | 2 +-
dev-support/bin/dist-tools-hooks-maker | 182 ++++++++
dev-support/docker/Dockerfile | 8 +
.../JWTRedirectAuthenticationHandler.java | 16 +-
.../TestJWTRedirectAuthentictionHandler.java | 37 +-
.../hadoop-common/src/main/bin/hadoop | 10 +-
.../src/main/bin/hadoop-functions.sh | 92 +++-
.../src/main/bin/hadoop-layout.sh.example | 10 +-
.../hadoop-common/src/main/conf/hadoop-env.sh | 23 +-
.../crypto/key/kms/KMSClientProvider.java | 8 +-
.../hadoop/crypto/key/kms/ValueQueue.java | 16 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 126 ++++-
.../java/org/apache/hadoop/net/NetUtils.java | 16 +-
.../org/apache/hadoop/net/NetworkTopology.java | 17 +-
.../java/org/apache/hadoop/net/NodeBase.java | 18 +-
.../src/org/apache/hadoop/io/compress/lz4/lz4.c | 2 +-
.../src/main/resources/core-default.xml | 2 +-
.../src/site/markdown/GroupsMapping.md | 26 +-
.../src/site/markdown/HttpAuthentication.md | 6 +-
.../hadoop/crypto/key/TestValueQueue.java | 6 -
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 192 +++++++-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 395 ++++------------
.../hadoop/security/TestDoAsEffectiveUser.java | 6 +-
.../hadoop-common/src/test/proto/test.proto | 9 +
.../src/test/proto/test_rpc_service.proto | 4 +
...apache.hadoop.security.token.TokenIdentifier | 2 +-
.../scripts/hadoop_add_to_classpath_tools.bats | 47 ++
.../hadoop_add_to_classpath_toolspath.bats | 74 ---
.../src/test/scripts/hadoop_basic_init.bats | 3 +-
.../src/test/scripts/hadoop_bootstrap.bats | 15 +-
.../src/test/scripts/hadoop_entry_tests.bats | 49 ++
.../src/test/scripts/hadoop_shellprofile.bats | 7 +
.../hadoop/crypto/key/kms/server/KMS.java | 14 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 42 ++
hadoop-dist/pom.xml | 18 +
.../dev-support/findbugsExcludeFile.xml | 1 +
.../org/apache/hadoop/hdfs/BlockReader.java | 10 +-
.../apache/hadoop/hdfs/BlockReaderFactory.java | 7 +-
.../apache/hadoop/hdfs/BlockReaderLocal.java | 10 +-
.../hadoop/hdfs/BlockReaderLocalLegacy.java | 10 +-
.../org/apache/hadoop/hdfs/ClientContext.java | 56 ++-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 22 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 30 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 3 -
.../apache/hadoop/hdfs/ExternalBlockReader.java | 10 +-
.../apache/hadoop/hdfs/FileChecksumHelper.java | 187 +++++++-
.../apache/hadoop/hdfs/RemoteBlockReader.java | 29 +-
.../apache/hadoop/hdfs/RemoteBlockReader2.java | 29 +-
.../org/apache/hadoop/hdfs/ReplicaAccessor.java | 7 +
.../hdfs/protocol/DatanodeAdminProperties.java | 100 ++++
.../apache/hadoop/hdfs/protocol/DatanodeID.java | 6 +
.../hadoop/hdfs/protocol/StripedBlockInfo.java | 61 +++
.../datatransfer/DataTransferProtocol.java | 16 +-
.../hadoop/hdfs/protocol/datatransfer/Op.java | 1 +
.../hdfs/protocol/datatransfer/Sender.java | 19 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 42 +-
.../hdfs/util/CombinedHostsFileReader.java | 76 +++
.../hdfs/util/CombinedHostsFileWriter.java | 69 +++
.../hadoop/hdfs/util/StripedBlockUtil.java | 12 +
.../src/main/proto/datatransfer.proto | 9 +-
.../src/main/native/fuse-dfs/fuse_connect.c | 5 +
.../hadoop-hdfs/src/main/bin/hdfs | 5 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +-
.../hdfs/protocol/datatransfer/Receiver.java | 28 ++
.../CombinedHostFileManager.java | 250 ++++++++++
.../server/blockmanagement/DatanodeManager.java | 59 ++-
.../blockmanagement/HostConfigManager.java | 80 ++++
.../server/blockmanagement/HostFileManager.java | 147 ++----
.../hdfs/server/blockmanagement/HostSet.java | 114 +++++
.../server/datanode/BlockChecksumHelper.java | 284 +++++++++---
.../hdfs/server/datanode/BlockReceiver.java | 2 +-
.../hdfs/server/datanode/DataXceiver.java | 43 ++
.../erasurecode/ErasureCodingWorker.java | 3 +-
.../server/namenode/EncryptionZoneManager.java | 15 +
.../server/namenode/FSDirEncryptionZoneOp.java | 83 ++++
.../hdfs/server/namenode/FSNamesystem.java | 27 ++
.../OfflineImageReconstructor.java | 312 +++++++------
.../offlineImageViewer/PBImageXmlWriter.java | 463 +++++++++++++------
.../src/main/resources/hdfs-default.xml | 32 ++
.../src/main/webapps/hdfs/dfshealth.html | 13 +-
.../src/site/markdown/HdfsUserGuide.md | 6 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 61 +--
.../hadoop/fs/TestEnhancedByteBufferAccess.java | 4 +-
.../hadoop/hdfs/TestBlockReaderLocal.java | 4 +-
.../org/apache/hadoop/hdfs/TestConnCache.java | 2 -
.../apache/hadoop/hdfs/TestDatanodeReport.java | 57 ++-
.../hadoop/hdfs/TestDistributedFileSystem.java | 62 +++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 4 +-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 36 +-
.../hadoop/hdfs/TestExternalBlockReader.java | 8 +-
.../apache/hadoop/hdfs/TestFileChecksum.java | 247 ++++++++++
.../TestBlocksWithNotEnoughRacks.java | 34 +-
.../blockmanagement/TestDatanodeManager.java | 8 +-
.../blockmanagement/TestHostFileManager.java | 10 +-
.../hdfs/server/namenode/TestHostsFiles.java | 70 ++-
.../server/namenode/TestNameNodeMXBean.java | 25 +-
.../hdfs/server/namenode/TestStartup.java | 54 +--
.../TestUpgradeDomainBlockPlacementPolicy.java | 169 +++++++
.../hadoop/hdfs/util/HostsFileWriter.java | 122 +++++
.../hdfs/util/TestCombinedHostsFileReader.java | 79 ++++
.../apache/hadoop/net/TestNetworkTopology.java | 7 +
.../src/test/resources/dfs.hosts.json | 5 +
hadoop-mapreduce-project/bin/mapred | 10 +-
.../org/apache/hadoop/mapreduce/TaskID.java | 57 ++-
.../org/apache/hadoop/mapreduce/tools/CLI.java | 48 +-
.../hadoop/mapreduce/util/ProcessTree.java | 7 +-
.../src/site/markdown/EncryptedShuffle.md | 2 +-
.../org/apache/hadoop/mapreduce/TestTaskID.java | 461 ++++++++++++++++++
.../apache/hadoop/mapreduce/tools/TestCLI.java | 38 +-
.../java/org/apache/hadoop/hdfs/NNBench.java | 1 +
.../mapreduce/v2/TestMRJobsWithProfiler.java | 2 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 17 +
hadoop-tools/hadoop-archives/pom.xml | 17 +
hadoop-tools/hadoop-aws/pom.xml | 17 +
.../src/site/markdown/tools/hadoop-aws/index.md | 5 +-
hadoop-tools/hadoop-azure/pom.xml | 18 +
.../hadoop-azure/src/site/markdown/index.md | 3 +
hadoop-tools/hadoop-datajoin/pom.xml | 16 +
hadoop-tools/hadoop-distcp/pom.xml | 11 +
.../java/org/apache/hadoop/tools/DistCp.java | 79 ----
.../apache/hadoop/tools/DistCpConstants.java | 16 -
.../apache/hadoop/tools/DistCpOptionSwitch.java | 9 -
.../org/apache/hadoop/tools/DistCpOptions.java | 22 -
.../org/apache/hadoop/tools/OptionsParser.java | 5 -
.../apache/hadoop/tools/mapred/CopyMapper.java | 68 ---
.../src/site/markdown/DistCp.md.vm | 39 --
.../apache/hadoop/tools/TestOptionsParser.java | 17 +-
hadoop-tools/hadoop-extras/pom.xml | 17 +
hadoop-tools/hadoop-gridmix/pom.xml | 17 +
hadoop-tools/hadoop-kafka/pom.xml | 17 +
hadoop-tools/hadoop-openstack/pom.xml | 17 +
.../hadoop-openstack/src/site/markdown/index.md | 7 +-
hadoop-tools/hadoop-rumen/pom.xml | 17 +
hadoop-tools/hadoop-sls/pom.xml | 17 +
.../hadoop-sls/src/main/bin/rumen2sls.sh | 2 +-
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 2 +-
hadoop-tools/hadoop-streaming/pom.xml | 17 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +
.../hadoop/yarn/client/TestRMFailover.java | 7 +-
.../yarn/client/api/impl/TestAMRMProxy.java | 413 +++++++++++++++++
.../api/impl/FileSystemTimelineWriter.java | 106 ++++-
.../yarn/util/ProcfsBasedProcessTree.java | 6 +-
.../util/ResourceCalculatorProcessTree.java | 8 +-
.../yarn/util/WindowsBasedProcessTree.java | 5 +-
.../yarn/util/TestProcfsBasedProcessTree.java | 8 -
.../util/TestResourceCalculatorProcessTree.java | 3 +-
.../yarn/util/TestWindowsBasedProcessTree.java | 3 +-
.../nodemanager/amrmproxy/AMRMProxyService.java | 14 +-
.../amrmproxy/DefaultRequestInterceptor.java | 7 +
.../containermanager/ContainerManagerImpl.java | 49 +-
.../server/resourcemanager/ClientRMService.java | 9 +-
.../resourcemanager/webapp/RMWebAppFilter.java | 40 +-
.../webapp/dao/ReservationIdInfo.java | 64 ---
.../webapp/dao/ReservationInfo.java | 7 +-
.../ParameterizedSchedulerTestBase.java | 36 +-
.../yarn/server/resourcemanager/TestRM.java | 4 -
.../server/resourcemanager/TestRMRestart.java | 4 -
.../TestWorkPreservingRMRestart.java | 9 -
.../reservation/TestReservationSystem.java | 4 -
.../scheduler/TestAbstractYarnScheduler.java | 6 -
.../security/TestClientToAMTokens.java | 4 -
.../webapp/TestRMWebServicesReservation.java | 15 +-
.../src/test/resources/test-fair-scheduler.xml | 34 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 75 ++-
.../src/site/markdown/CapacityScheduler.md | 12 +
.../src/site/markdown/FairScheduler.md | 6 +-
.../src/site/markdown/ResourceManagerRest.md | 217 +++++++++
168 files changed, 5848 insertions(+), 1821 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30473ecf/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
index 30e946d,af720c7..6f5f1e3
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
@@@ -22,8 -22,9 +22,10 @@@ import org.apache.hadoop.classification
import org.apache.hadoop.classification.InterfaceStability;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+ import java.net.InetSocketAddress;
+
/**
* This class represents the primary identifier for a Datanode.
* Datanodes are identified by how they can be contacted (hostname
@@@ -277,54 -277,7 +279,58 @@@ public class DatanodeID implements Comp
return getXferAddr().compareTo(that.getXferAddr());
}
+ /**
+ * Returns the container port.
+ * @return Port
+ */
+ public int getContainerPort() {
+ return containerPort;
+ }
+
+ /**
+ * Sets the container port.
+ * @param containerPort - container port.
+ */
+ public void setContainerPort(int containerPort) {
+ this.containerPort = containerPort;
+ }
+
+ /**
+ * Returns a DataNode ID from the protocol buffers.
+ *
+ * @param datanodeIDProto - protoBuf Message
+ * @return DataNodeID
+ */
+ public static DatanodeID getFromProtoBuf(HdfsProtos.DatanodeIDProto
+ datanodeIDProto) {
+ DatanodeID id = new DatanodeID(datanodeIDProto.getDatanodeUuid(),
+ datanodeIDProto.getIpAddr(), datanodeIDProto.getHostName(),
+ datanodeIDProto.getXferPort(), datanodeIDProto.getInfoPort(),
+ datanodeIDProto.getInfoSecurePort(), datanodeIDProto.getIpcPort());
+ id.setContainerPort(datanodeIDProto.getContainerPort());
+ return id;
+ }
+
+ /**
+ * Returns a DataNodeID protobuf message from a datanode ID.
+ * @return HdfsProtos.DatanodeIDProto
+ */
+ public HdfsProtos.DatanodeIDProto getProtoBufMessage() {
+ HdfsProtos.DatanodeIDProto.Builder builder =
+ HdfsProtos.DatanodeIDProto.newBuilder();
+
+ return builder.setDatanodeUuid(this.getDatanodeUuid())
+ .setIpAddr(this.getIpcAddr())
+ .setHostName(this.getHostName())
+ .setXferPort(this.getXferPort())
+ .setInfoPort(this.getInfoPort())
+ .setInfoSecurePort(this.getInfoSecurePort())
+ .setIpcPort(this.getIpcPort())
+ .setContainerPort(this.getContainerPort())
+ .build();
+ }
++
+ public InetSocketAddress getResolvedAddress() {
+ return new InetSocketAddress(this.getIpAddr(), this.getXferPort());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30473ecf/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------