You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/01/07 23:42:07 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-1312
Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9af2370
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9af2370
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9af2370
Branch: refs/heads/HDFS-1312
Commit: a9af23708827848d841d90c5d3c8bacefc50a3d2
Parents: 1bffbf6 89022f8
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Jan 7 14:40:23 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Jan 7 14:40:23 2016 -0800
----------------------------------------------------------------------
.../authentication/util/KerberosUtil.java | 87 +-
.../authentication/util/TestKerberosUtil.java | 31 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 117 +-
hadoop-common-project/hadoop-common/pom.xml | 17 +-
.../hadoop-common/src/CMakeLists.txt | 3 +-
.../crypto/key/kms/KMSClientProvider.java | 14 +-
.../hadoop/fs/CommonConfigurationKeys.java | 5 +
.../fs/CommonConfigurationKeysPublic.java | 6 +
.../java/org/apache/hadoop/fs/FileSystem.java | 104 +-
.../org/apache/hadoop/fs/FilterFileSystem.java | 11 +
.../main/java/org/apache/hadoop/fs/FsShell.java | 12 +-
.../main/java/org/apache/hadoop/fs/Trash.java | 14 +-
.../java/org/apache/hadoop/fs/TrashPolicy.java | 56 +-
.../apache/hadoop/fs/TrashPolicyDefault.java | 193 ++-
.../apache/hadoop/fs/shell/CopyCommands.java | 40 +-
.../org/apache/hadoop/http/HttpServer2.java | 8 +-
.../main/java/org/apache/hadoop/io/IOUtils.java | 5 +-
.../rawcoder/AbstractRawErasureDecoder.java | 55 +-
.../rawcoder/AbstractRawErasureEncoder.java | 56 +-
.../erasurecode/rawcoder/RawErasureDecoder.java | 5 +-
.../erasurecode/rawcoder/RawErasureEncoder.java | 5 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 4 +-
.../apache/hadoop/ipc/DecayRpcScheduler.java | 2 +-
.../main/java/org/apache/hadoop/ipc/RPC.java | 19 +-
.../apache/hadoop/net/unix/DomainSocket.java | 59 +-
.../hadoop/security/NullGroupsMapping.java | 50 +
.../apache/hadoop/security/SecurityUtil.java | 6 +
.../hadoop/security/UserGroupInformation.java | 36 +
.../org/apache/hadoop/util/JvmPauseMonitor.java | 50 +-
.../hadoop/util/NativeLibraryChecker.java | 10 +-
.../org/apache/hadoop/util/StringUtils.java | 19 +-
.../java/org/apache/hadoop/util/ThreadUtil.java | 30 +
.../org/apache/hadoop/util/VersionInfo.java | 6 +-
.../src/main/resources/core-default.xml | 39 +
.../src/site/markdown/FileSystemShell.md | 31 +-
.../src/site/markdown/HttpAuthentication.md | 52 +-
.../src/site/markdown/SecureMode.md | 303 ++--
.../src/site/markdown/SingleCluster.md.vm | 4 +-
.../conf/TestConfigurationFieldsBase.java | 14 +-
.../hadoop/fs/TestFileSystemInitialization.java | 16 +
.../org/apache/hadoop/fs/TestFsShellCopy.java | 24 +-
.../org/apache/hadoop/fs/TestHarFileSystem.java | 4 +
.../java/org/apache/hadoop/fs/TestTrash.java | 9 +
.../apache/hadoop/fs/shell/find/TestFind.java | 53 +-
.../org/apache/hadoop/http/TestHttpServer.java | 3 -
.../apache/hadoop/http/TestHttpServerLogs.java | 80 +
.../org/apache/hadoop/io/TestSequenceFile.java | 17 +-
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 7 +
.../erasurecode/rawcoder/TestRawCoderBase.java | 34 +
.../ipc/TestProtoBufRPCCompatibility.java | 178 ++
.../java/org/apache/hadoop/ipc/TestRPC.java | 30 +
.../metrics2/impl/TestGangliaMetrics.java | 10 +-
.../metrics2/impl/TestMetricsSystemImpl.java | 1 +
.../hadoop/metrics2/source/TestJvmMetrics.java | 70 +-
.../hadoop/security/TestNullGroupsMapping.java | 60 +
.../apache/hadoop/test/GenericTestUtils.java | 54 +-
.../hadoop-common/src/test/proto/test.proto | 8 +
.../src/test/proto/test_rpc_service.proto | 15 +
.../src/test/resources/testConf.xml | 7 +-
.../hadoop-kms/src/main/conf/kms-site.xml | 6 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 28 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 21 +-
.../crypto/key/kms/server/TestKMSAudit.java | 14 +-
.../java/org/apache/hadoop/minikdc/MiniKdc.java | 54 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 +
.../apache/hadoop/hdfs/BlockReaderFactory.java | 2 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 28 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 47 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 1 +
.../hadoop/hdfs/DistributedFileSystem.java | 62 +
.../org/apache/hadoop/hdfs/protocol/Block.java | 7 +-
.../hdfs/protocol/LocatedStripedBlock.java | 10 +-
.../hdfs/protocol/ReconfigurationProtocol.java | 54 +
.../ClientDatanodeProtocolTranslatorPB.java | 46 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 43 +-
.../protocolPB/ReconfigurationProtocolPB.java | 43 +
.../ReconfigurationProtocolTranslatorPB.java | 146 ++
.../ReconfigurationProtocolUtils.java | 64 +
.../hadoop/hdfs/protocolPB/package-info.java | 18 +
.../hdfs/shortcircuit/DfsClientShmManager.java | 2 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 365 ++++-
.../src/main/proto/ClientDatanodeProtocol.proto | 32 +-
.../main/proto/ReconfigurationProtocol.proto | 74 +
.../src/main/proto/erasurecoding.proto | 2 +-
.../src/main/proto/hdfs.proto | 2 +-
.../hadoop-hdfs-native-client/pom.xml | 5 +-
.../src/CMakeLists.txt | 5 -
.../src/contrib/libwebhdfs/CMakeLists.txt | 88 -
.../libwebhdfs/resources/FindJansson.cmake | 43 -
.../contrib/libwebhdfs/src/hdfs_http_client.c | 490 ------
.../contrib/libwebhdfs/src/hdfs_http_client.h | 294 ----
.../contrib/libwebhdfs/src/hdfs_http_query.c | 402 -----
.../contrib/libwebhdfs/src/hdfs_http_query.h | 240 ---
.../contrib/libwebhdfs/src/hdfs_json_parser.c | 654 --------
.../contrib/libwebhdfs/src/hdfs_json_parser.h | 178 --
.../src/contrib/libwebhdfs/src/hdfs_web.c | 1538 ------------------
.../libwebhdfs/src/test_libwebhdfs_ops.c | 552 -------
.../libwebhdfs/src/test_libwebhdfs_read.c | 78 -
.../libwebhdfs/src/test_libwebhdfs_threaded.c | 247 ---
.../libwebhdfs/src/test_libwebhdfs_write.c | 111 --
.../src/main/native/fuse-dfs/doc/README | 26 +-
.../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 34 +-
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 3 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 213 ++-
.../bkjournal/TestBootstrapStandbyWithBKJM.java | 1 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
...tDatanodeProtocolServerSideTranslatorPB.java | 58 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 5 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 38 +-
...igurationProtocolServerSideTranslatorPB.java | 89 +
.../ReconfigurationProtocolServerSideUtils.java | 78 +
.../server/blockmanagement/BlockIdManager.java | 4 +-
.../blockmanagement/BlockInfoStriped.java | 2 +-
.../server/blockmanagement/BlockManager.java | 397 +++--
.../blockmanagement/BlockManagerSafeMode.java | 638 ++++++++
.../BlockPlacementPolicyDefault.java | 15 +-
.../BlockUnderConstructionFeature.java | 4 +-
.../blockmanagement/DatanodeDescriptor.java | 24 +-
.../server/blockmanagement/DatanodeManager.java | 432 ++---
.../server/blockmanagement/DatanodeStats.java | 19 +-
.../blockmanagement/DatanodeStorageInfo.java | 2 +-
.../blockmanagement/DecommissionManager.java | 39 +-
.../blockmanagement/ErasureCodingWork.java | 6 +-
.../blockmanagement/HeartbeatManager.java | 8 +-
.../hdfs/server/datanode/BPServiceActor.java | 4 +
.../datanode/BPServiceActorActionException.java | 5 +
.../server/datanode/BlockRecoveryWorker.java | 238 ++-
.../hadoop/hdfs/server/datanode/DataNode.java | 64 +-
.../hdfs/server/datanode/DataXceiver.java | 32 +-
.../hdfs/server/datanode/DataXceiverServer.java | 58 +-
.../hdfs/server/datanode/ErrorReportAction.java | 2 +-
.../hdfs/server/datanode/ReplicaInfo.java | 79 +
.../server/datanode/ReportBadBlockAction.java | 2 +-
.../server/datanode/ShortCircuitRegistry.java | 14 +-
.../erasurecode/ErasureCodingWorker.java | 2 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 121 +-
.../hdfs/server/namenode/CacheManager.java | 2 +-
.../hdfs/server/namenode/Checkpointer.java | 3 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 24 +-
.../hdfs/server/namenode/FSDirDeleteOp.java | 2 +-
.../hdfs/server/namenode/FSDirRenameOp.java | 3 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 2 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 2 +-
.../hdfs/server/namenode/FSDirectory.java | 49 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../server/namenode/FSImageFormatPBINode.java | 1 +
.../hdfs/server/namenode/FSNamesystem.java | 1135 ++-----------
.../hadoop/hdfs/server/namenode/NNStorage.java | 18 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 29 +-
.../server/namenode/NameNodeHttpServer.java | 9 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 43 +-
.../hdfs/server/namenode/NamenodeFsck.java | 16 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 18 +-
.../hadoop/hdfs/server/namenode/SafeMode.java | 18 -
.../hdfs/server/namenode/SecondaryNameNode.java | 14 +-
.../server/namenode/ha/BootstrapStandby.java | 6 +-
.../namenode/metrics/NameNodeMetrics.java | 12 +
.../server/protocol/BlockECRecoveryCommand.java | 11 +-
.../server/protocol/BlockRecoveryCommand.java | 33 +
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
.../hadoop-hdfs/src/main/proto/HdfsServer.proto | 6 +-
.../src/main/resources/hdfs-default.xml | 132 ++
.../src/main/webapps/hdfs/explorer.html | 9 +-
.../src/main/webapps/hdfs/explorer.js | 30 +
.../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 163 +-
.../src/site/markdown/HdfsMultihoming.md | 42 +
.../src/site/markdown/HdfsPermissionsGuide.md | 72 +-
.../org/apache/hadoop/TestRefreshCallQueue.java | 1 +
.../java/org/apache/hadoop/cli/TestAclCLI.java | 2 +
.../apache/hadoop/cli/TestCacheAdminCLI.java | 2 +
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 2 +
.../org/apache/hadoop/cli/TestDeleteCLI.java | 2 +
.../apache/hadoop/cli/TestErasureCodingCLI.java | 2 +
.../java/org/apache/hadoop/cli/TestHDFSCLI.java | 2 +
.../org/apache/hadoop/cli/TestXAttrCLI.java | 2 +
.../apache/hadoop/fs/TestFcHdfsCreateMkdir.java | 4 +-
.../apache/hadoop/fs/TestFcHdfsPermission.java | 4 +-
.../apache/hadoop/fs/TestFcHdfsSetUMask.java | 4 +-
.../org/apache/hadoop/fs/TestSymlinkHdfs.java | 4 +-
.../hadoop/fs/contract/hdfs/HDFSContract.java | 1 +
.../hadoop/fs/shell/TestHdfsTextCommand.java | 2 +
.../fs/viewfs/TestViewFileSystemHdfs.java | 4 +-
.../fs/viewfs/TestViewFileSystemWithAcls.java | 4 +-
.../fs/viewfs/TestViewFileSystemWithXAttrs.java | 4 +-
.../hadoop/fs/viewfs/TestViewFsAtHdfsRoot.java | 4 +-
.../apache/hadoop/fs/viewfs/TestViewFsHdfs.java | 4 +-
.../hadoop/fs/viewfs/TestViewFsWithAcls.java | 4 +-
.../hadoop/fs/viewfs/TestViewFsWithXAttrs.java | 4 +-
.../org/apache/hadoop/hdfs/FileAppendTest4.java | 4 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 41 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 16 +-
.../apache/hadoop/hdfs/TestAbandonBlock.java | 10 +-
.../hadoop/hdfs/TestBlocksScheduledCounter.java | 2 +
.../TestClientProtocolForPipelineRecovery.java | 4 +-
.../hadoop/hdfs/TestClientReportBadBlock.java | 10 +-
.../hadoop/hdfs/TestDFSClientExcludedNodes.java | 1 +
.../hadoop/hdfs/TestDFSClientFailover.java | 1 +
.../hadoop/hdfs/TestDFSClientSocketSize.java | 1 +
.../org/apache/hadoop/hdfs/TestDFSFinalize.java | 5 +-
.../apache/hadoop/hdfs/TestDFSOutputStream.java | 4 +-
.../apache/hadoop/hdfs/TestDFSPermission.java | 1 +
.../org/apache/hadoop/hdfs/TestDFSRollback.java | 5 +-
.../hadoop/hdfs/TestDFSStartupVersions.java | 5 +-
.../hdfs/TestDFSStorageStateRecovery.java | 5 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 1 +
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 1 +
.../hadoop/hdfs/TestDataTransferKeepalive.java | 5 +-
.../hadoop/hdfs/TestDatanodeRegistration.java | 140 +-
.../apache/hadoop/hdfs/TestDecommission.java | 1 +
.../hadoop/hdfs/TestDistributedFileSystem.java | 28 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 86 +
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 1 +
.../hadoop/hdfs/TestErasureCodingPolicies.java | 1 +
.../TestErasureCodingPolicyWithSnapshot.java | 1 +
.../org/apache/hadoop/hdfs/TestFileAppend.java | 66 +
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 18 +-
.../hadoop/hdfs/TestFileConcurrentReader.java | 6 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 6 +-
.../org/apache/hadoop/hdfs/TestFileStatus.java | 8 +-
.../hadoop/hdfs/TestFileStatusWithECPolicy.java | 1 +
.../apache/hadoop/hdfs/TestGetFileChecksum.java | 1 +
.../hadoop/hdfs/TestHDFSFileSystemContract.java | 6 +-
.../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 1 +
.../apache/hadoop/hdfs/TestLeaseRecovery.java | 1 +
.../apache/hadoop/hdfs/TestLeaseRecovery2.java | 6 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 273 ++++
.../apache/hadoop/hdfs/TestListFilesInDFS.java | 6 +-
.../hadoop/hdfs/TestListFilesInFileContext.java | 4 +-
.../apache/hadoop/hdfs/TestMiniDFSCluster.java | 19 +-
.../org/apache/hadoop/hdfs/TestPipelines.java | 5 +-
.../hdfs/TestReadStripedFileWithDecoding.java | 1 +
.../TestReadStripedFileWithMissingBlocks.java | 1 +
.../hadoop/hdfs/TestRecoverStripedFile.java | 5 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 1 +
.../apache/hadoop/hdfs/TestRollingUpgrade.java | 1 +
.../hadoop/hdfs/TestRollingUpgradeRollback.java | 1 +
.../org/apache/hadoop/hdfs/TestSafeMode.java | 4 +-
.../hdfs/TestSafeModeWithStripedFile.java | 1 +
.../org/apache/hadoop/hdfs/TestWriteRead.java | 5 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 1 +
.../datatransfer/sasl/TestSaslDataTransfer.java | 1 +
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 15 +-
.../hdfs/qjournal/MiniJournalCluster.java | 38 +
.../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 1 +
.../hdfs/qjournal/TestMiniJournalCluster.java | 1 +
.../hadoop/hdfs/qjournal/TestNNWithQJM.java | 1 +
.../hdfs/qjournal/TestSecureNNWithQJM.java | 3 +
.../qjournal/client/TestEpochsAreUnique.java | 1 +
.../hdfs/qjournal/client/TestQJMWithFaults.java | 3 +
.../client/TestQuorumJournalManager.java | 15 +-
.../qjournal/server/TestJournalNodeMXBean.java | 11 +-
.../hdfs/security/TestDelegationToken.java | 1 +
.../BaseReplicationPolicyTest.java | 2 +
.../blockmanagement/BlockManagerTestUtil.java | 11 +-
.../blockmanagement/TestBlockManager.java | 263 ++-
.../TestBlockManagerSafeMode.java | 480 ++++++
.../blockmanagement/TestBlockStatsMXBean.java | 63 +
.../TestBlocksWithNotEnoughRacks.java | 8 +-
.../TestComputeInvalidateWork.java | 2 +
.../TestPendingInvalidateBlock.java | 1 +
.../blockmanagement/TestPendingReplication.java | 4 +-
.../blockmanagement/TestReplicationPolicy.java | 56 +-
.../TestReplicationPolicyWithNodeGroup.java | 32 +
.../TestSequentialBlockGroupId.java | 1 +
.../server/datanode/BlockReportTestBase.java | 12 +-
.../hdfs/server/datanode/DataNodeTestUtils.java | 4 -
.../server/datanode/FsDatasetTestUtils.java | 36 +-
.../server/datanode/SimulatedFSDataset.java | 3 +-
.../server/datanode/TestBPOfferService.java | 32 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 34 +-
.../hdfs/server/datanode/TestDataNodeExit.java | 4 +-
.../datanode/TestDataNodeReconfiguration.java | 259 +++
.../datanode/TestDataNodeVolumeFailure.java | 3 +
.../TestDataNodeVolumeFailureReporting.java | 1 +
.../TestDataNodeVolumeFailureToleration.java | 1 +
.../server/datanode/TestDeleteBlockPool.java | 69 +-
.../server/datanode/TestDirectoryScanner.java | 4 +-
.../hdfs/server/datanode/TestDiskError.java | 14 +-
.../server/datanode/TestFsDatasetCache.java | 2 +
.../datanode/TestIncrementalBrVariations.java | 19 +-
.../server/datanode/TestSimulatedFSDataset.java | 47 +
.../fsdataset/impl/FsDatasetImplTestUtils.java | 81 +
.../fsdataset/impl/FsDatasetTestUtil.java | 10 +-
.../fsdataset/impl/LazyPersistTestCase.java | 2 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 4 +
.../hdfs/server/namenode/FSImageTestUtil.java | 7 +
.../hdfs/server/namenode/FSXAttrBaseTest.java | 2 +-
.../server/namenode/NNThroughputBenchmark.java | 10 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 12 +-
.../hdfs/server/namenode/TestAclConfigFlag.java | 1 +
.../hdfs/server/namenode/TestAddBlock.java | 1 +
.../hdfs/server/namenode/TestAddBlockRetry.java | 1 +
.../TestAddOverReplicatedStripedBlocks.java | 1 +
.../server/namenode/TestAddStripedBlocks.java | 9 +-
.../hdfs/server/namenode/TestAuditLogs.java | 12 +-
.../hdfs/server/namenode/TestBackupNode.java | 14 +-
...stBlockPlacementPolicyRackFaultTolerant.java | 1 +
.../server/namenode/TestCacheDirectives.java | 1 +
.../hdfs/server/namenode/TestCheckpoint.java | 2 +-
.../TestCommitBlockSynchronization.java | 5 +-
.../TestCommitBlockWithInvalidGenStamp.java | 1 +
.../hdfs/server/namenode/TestDeadDatanode.java | 21 +-
.../TestDefaultBlockPlacementPolicy.java | 1 +
.../namenode/TestDiskspaceQuotaUpdate.java | 1 +
.../server/namenode/TestEditLogAutoroll.java | 2 +
.../namenode/TestEditLogJournalFailures.java | 5 +-
.../hdfs/server/namenode/TestEditLogRace.java | 2 +-
.../hdfs/server/namenode/TestFSDirectory.java | 1 +
.../server/namenode/TestFSEditLogLoader.java | 4 +-
.../server/namenode/TestFSImageWithAcl.java | 4 +-
.../namenode/TestFSImageWithSnapshot.java | 1 +
.../server/namenode/TestFSImageWithXAttr.java | 4 +-
.../hdfs/server/namenode/TestFSNamesystem.java | 13 +-
.../hdfs/server/namenode/TestFileTruncate.java | 10 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 6 +-
.../hdfs/server/namenode/TestHDFSConcat.java | 21 +
.../hdfs/server/namenode/TestHostsFiles.java | 8 +-
.../namenode/TestINodeAttributeProvider.java | 1 +
.../hdfs/server/namenode/TestINodeFile.java | 7 +-
.../hdfs/server/namenode/TestMalformedURLs.java | 1 +
.../namenode/TestMetadataVersionOutput.java | 1 +
.../TestNameNodeMetadataConsistency.java | 61 +-
.../namenode/TestNameNodeRetryCacheMetrics.java | 1 +
.../namenode/TestNameNodeRpcServerMethods.java | 86 +
.../server/namenode/TestNamenodeRetryCache.java | 5 +-
.../server/namenode/TestQuotaByStorageType.java | 1 +
.../namenode/TestQuotaWithStripedBlocks.java | 1 +
.../namenode/TestRecoverStripedBlocks.java | 5 +-
.../hdfs/server/namenode/TestStartup.java | 2 +
.../server/namenode/TestXAttrConfigFlag.java | 1 +
.../hdfs/server/namenode/ha/HATestUtil.java | 2 +-
.../namenode/ha/TestBootstrapStandby.java | 12 +
.../ha/TestBootstrapStandbyWithQJM.java | 2 +
.../hdfs/server/namenode/ha/TestDNFencing.java | 1 +
.../namenode/ha/TestDelegationTokensWithHA.java | 1 +
.../ha/TestFailoverWithBlockTokensEnabled.java | 1 +
.../namenode/ha/TestFailureToReadEdits.java | 3 +
.../server/namenode/ha/TestGetGroupsWithHA.java | 1 +
.../hdfs/server/namenode/ha/TestHASafeMode.java | 12 +-
.../namenode/ha/TestInitializeSharedEdits.java | 1 +
.../server/namenode/ha/TestNNHealthCheck.java | 1 +
.../ha/TestPendingCorruptDnMessages.java | 2 +-
.../namenode/ha/TestPipelinesFailover.java | 38 +-
.../server/namenode/ha/TestQuotasWithHA.java | 1 +
.../namenode/ha/TestRetryCacheWithHA.java | 1 +
.../namenode/ha/TestSeveralNameNodes.java | 8 +-
.../namenode/ha/TestStandbyCheckpoints.java | 1 +
.../server/namenode/ha/TestXAttrsWithHA.java | 1 +
.../TestNNMetricFilesInGetListingOps.java | 5 +-
.../namenode/metrics/TestNameNodeMetrics.java | 5 +-
.../snapshot/TestFileContextSnapshot.java | 1 +
.../TestGetContentSummaryWithSnapshot.java | 1 +
...tINodeFileUnderConstructionWithSnapshot.java | 1 +
.../namenode/snapshot/TestNestedSnapshots.java | 1 +
.../snapshot/TestOpenFilesWithSnapshot.java | 2 +
.../snapshot/TestRenameWithSnapshots.java | 1 +
.../snapshot/TestSetQuotaWithSnapshot.java | 1 +
.../server/namenode/snapshot/TestSnapshot.java | 1 +
.../snapshot/TestSnapshotBlocksMap.java | 1 +
.../namenode/snapshot/TestSnapshotDeletion.java | 1 +
.../snapshot/TestSnapshotDiffReport.java | 1 +
.../namenode/snapshot/TestSnapshotListing.java | 1 +
.../namenode/snapshot/TestSnapshotMetrics.java | 1 +
.../namenode/snapshot/TestSnapshotRename.java | 1 +
.../snapshot/TestSnapshotReplication.java | 1 +
.../snapshot/TestSnapshottableDirListing.java | 1 +
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +-
.../hdfs/tools/TestDFSHAAdminMiniCluster.java | 5 +-
.../hdfs/tools/TestDFSZKFailoverController.java | 8 +-
.../apache/hadoop/hdfs/tools/TestGetGroups.java | 5 +-
.../hdfs/tools/TestStoragePolicyCommands.java | 2 +
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 4 +-
.../hadoop/hdfs/web/TestHttpsFileSystem.java | 4 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 158 ++
.../hdfs/web/TestWebHdfsFileSystemContract.java | 2 +-
.../hadoop/hdfs/web/TestWebHdfsTokens.java | 6 +-
.../TestWebHdfsWithAuthenticationFilter.java | 8 +-
.../security/TestRefreshUserMappings.java | 2 +
.../org/apache/hadoop/tools/TestJMXGet.java | 17 +-
.../org/apache/hadoop/tracing/TestTracing.java | 5 +-
hadoop-mapreduce-project/CHANGES.txt | 40 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 13 +-
.../mapreduce/v2/app/webapp/TaskPage.java | 2 +-
.../mapreduce/v2/app/webapp/TasksPage.java | 2 +-
.../jobhistory/TestJobHistoryEventHandler.java | 14 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 21 +-
.../org/apache/hadoop/mapreduce/tools/CLI.java | 41 +-
.../src/main/resources/mapred-default.xml | 162 +-
.../src/site/markdown/MapReduceTutorial.md | 6 +-
.../org/apache/hadoop/mapred/TestTaskLog.java | 22 +-
.../apache/hadoop/mapreduce/tools/TestCLI.java | 31 +-
.../mapreduce/v2/hs/HistoryFileManager.java | 37 +-
.../mapreduce/v2/hs/JobHistoryServer.java | 7 +-
.../mapreduce/v2/hs/webapp/HsTaskPage.java | 2 +-
.../mapreduce/v2/hs/webapp/HsTasksPage.java | 2 +-
.../mapreduce/v2/hs/TestJobHistoryServer.java | 1 -
.../mapreduce/v2/hs/webapp/TestHSWebApp.java | 26 +
.../hadoop/mapred/ResourceMgrDelegate.java | 4 +-
.../org/apache/hadoop/mapred/YARNRunner.java | 5 +-
.../hadoop/mapred/TestMiniMRChildTask.java | 68 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 47 +-
hadoop-project/pom.xml | 1 +
.../hadoop/fs/s3/Jets3tFileSystemStore.java | 8 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 34 +-
.../fs/azure/TestNativeAzureFileSystemLive.java | 22 +
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 2 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 2 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 5 +-
.../sls/scheduler/ResourceSchedulerWrapper.java | 8 +
hadoop-yarn-project/CHANGES.txt | 214 ++-
.../dev-support/findbugs-exclude.xml | 5 +
.../UpdateApplicationPriorityResponse.java | 18 +-
.../yarn/api/records/ContainerStatus.java | 24 +
.../hadoop/yarn/api/records/ExecutionType.java | 43 +
.../api/records/FinalApplicationStatus.java | 5 +-
.../hadoop/yarn/api/records/NodeReport.java | 25 +
.../hadoop/yarn/api/records/QueueInfo.java | 16 +-
.../yarn/api/records/ResourceUtilization.java | 150 ++
.../records/timeline/TimelineEntityGroupId.java | 163 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 109 +-
.../yarn/server/api/ContainerContext.java | 22 +
.../src/main/proto/yarn_protos.proto | 16 +
.../src/main/proto/yarn_service_protos.proto | 1 +
.../yarn/conf/TestYarnConfigurationFields.java | 5 +
.../hadoop/yarn/client/api/YarnClient.java | 4 +-
.../yarn/client/api/impl/YarnClientImpl.java | 4 +-
.../hadoop/yarn/client/cli/ApplicationCLI.java | 15 +-
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 97 +-
.../apache/hadoop/yarn/client/cli/QueueCLI.java | 6 +
.../hadoop/yarn/client/ProtocolHATestBase.java | 4 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 200 ++-
...UpdateApplicationPriorityResponsePBImpl.java | 63 +
.../records/impl/pb/ContainerStatusPBImpl.java | 31 +-
.../api/records/impl/pb/NodeReportPBImpl.java | 73 +
.../yarn/api/records/impl/pb/ProtoUtils.java | 12 +
.../api/records/impl/pb/QueueInfoPBImpl.java | 13 +
.../impl/pb/ResourceUtilizationPBImpl.java | 104 ++
.../hadoop/yarn/client/api/TimelineClient.java | 43 +
.../client/api/impl/DirectTimelineWriter.java | 66 +
.../api/impl/FileSystemTimelineWriter.java | 847 ++++++++++
.../client/api/impl/TimelineClientImpl.java | 124 +-
.../yarn/client/api/impl/TimelineWriter.java | 142 ++
.../hadoop/yarn/event/AsyncDispatcher.java | 7 +-
.../nodelabels/CommonNodeLabelsManager.java | 18 +-
.../nodelabels/FileSystemNodeLabelsStore.java | 65 +-
.../hadoop/yarn/nodelabels/NodeLabelsStore.java | 16 +-
.../NonAppendableFSNodeLabelStore.java | 134 ++
.../yarn/security/ContainerTokenIdentifier.java | 35 +-
.../yarn/util/AbstractLivelinessMonitor.java | 4 +
.../apache/hadoop/yarn/util/MonotonicClock.java | 46 +
.../apache/hadoop/yarn/util/SystemClock.java | 4 +
.../yarn/util/WindowsBasedProcessTree.java | 51 +-
.../hadoop/yarn/webapp/view/JQueryUI.java | 21 +-
.../src/main/proto/yarn_security_token.proto | 1 +
.../webapps/static/dt-sorting/natural.js | 54 +
.../src/main/resources/yarn-default.xml | 57 +
.../hadoop/yarn/api/TestPBImplRecords.java | 4 +-
.../yarn/api/TestTimelineEntityGroupId.java | 52 +
.../client/api/impl/TestTimelineClient.java | 68 +-
.../api/impl/TestTimelineClientForATS1_5.java | 225 +++
.../DummyCommonNodeLabelsManager.java | 6 +-
.../TestFileSystemNodeLabelsStore.java | 28 +-
.../yarn/security/TestYARNTokenIdentifier.java | 13 +-
.../yarn/util/TestWindowsBasedProcessTree.java | 25 +-
...pplicationHistoryManagerOnTimelineStore.java | 7 +
.../ApplicationHistoryServer.java | 9 +-
.../webapp/AppAttemptPage.java | 2 +-
.../webapp/AppPage.java | 2 +-
.../server/timeline/TimelineDataManager.java | 6 +-
.../timeline/webapp/TimelineWebServices.java | 2 +
.../TestApplicationHistoryServer.java | 1 -
.../webapp/TestAHSWebApp.java | 50 +-
.../webapp/TestTimelineWebServices.java | 21 +
.../webapp/TestTimelineWebServicesWithSSL.java | 21 +-
.../impl/pb/NMContainerStatusPBImpl.java | 14 +-
.../yarn/server/api/records/NodeStatus.java | 1 +
.../server/api/records/ResourceUtilization.java | 133 --
.../api/records/impl/pb/NodeStatusPBImpl.java | 5 +-
.../impl/pb/ResourceUtilizationPBImpl.java | 104 --
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +-
.../hadoop/yarn/server/webapp/WebPageUtils.java | 6 +-
.../main/proto/yarn_server_common_protos.proto | 5 -
.../server/nodemanager/ContainerExecutor.java | 24 +-
.../nodemanager/DockerContainerExecutor.java | 11 +-
.../yarn/server/nodemanager/NodeManager.java | 7 +-
.../server/nodemanager/NodeResourceMonitor.java | 2 +-
.../nodemanager/NodeResourceMonitorImpl.java | 2 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 7 +-
.../container/ContainerImpl.java | 7 +-
.../launcher/ContainerLaunch.java | 169 +-
.../linux/resources/CGroupsHandler.java | 9 +-
.../CGroupsMemoryResourceHandlerImpl.java | 156 ++
.../linux/resources/MemoryResourceHandler.java | 27 +
.../linux/resources/ResourceHandlerModule.java | 30 +-
.../localizer/ResourceLocalizationService.java | 2 +-
.../logaggregation/AppLogAggregatorImpl.java | 6 +-
.../monitor/ContainersMonitor.java | 2 +-
.../monitor/ContainersMonitorImpl.java | 2 +-
.../nodemanager/webapp/AllApplicationsPage.java | 9 +-
.../nodemanager/webapp/AllContainersPage.java | 8 +-
.../nodemanager/TestNodeManagerResync.java | 3 +
.../nodemanager/TestNodeManagerShutdown.java | 3 +
.../amrmproxy/MockResourceManagerFacade.java | 6 +-
.../container/TestContainer.java | 59 +-
.../launcher/TestContainerLaunch.java | 252 ++-
.../TestCGroupsMemoryResourceHandlerImpl.java | 172 ++
.../TestResourceLocalizationService.java | 2 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 11 +
.../server/resourcemanager/AdminService.java | 49 +-
.../server/resourcemanager/ClientRMService.java | 12 +-
.../server/resourcemanager/ClusterMetrics.java | 20 +-
.../resourcemanager/LeaderElectorService.java | 144 ++
.../resourcemanager/NMLivelinessMonitor.java | 3 +-
.../resourcemanager/NodesListManager.java | 22 +-
.../server/resourcemanager/RMAppManager.java | 29 +-
.../yarn/server/resourcemanager/RMContext.java | 4 +
.../server/resourcemanager/RMContextImpl.java | 11 +
.../server/resourcemanager/ResourceManager.java | 60 +-
.../resourcemanager/ResourceTrackerService.java | 5 +
.../metrics/AppAttemptFinishedEvent.java | 10 +-
.../metrics/SystemMetricsPublisher.java | 20 +-
.../recovery/FileSystemRMStateStore.java | 12 +
.../recovery/LeveldbRMStateStore.java | 32 +
.../recovery/MemoryRMStateStore.java | 13 +
.../recovery/NullRMStateStore.java | 6 +
.../resourcemanager/recovery/RMStateStore.java | 63 +-
.../recovery/RMStateStoreEventType.java | 1 +
.../RMStateStoreRemoveAppAttemptEvent.java | 37 +
.../recovery/ZKRMStateStore.java | 16 +
.../recovery/records/ApplicationStateData.java | 10 +
.../reservation/CapacityOverTimePolicy.java | 52 +-
.../reservation/InMemoryPlan.java | 123 +-
.../InMemoryReservationAllocation.java | 13 +-
.../reservation/NoOverCommitPolicy.java | 8 +
.../resourcemanager/reservation/PlanView.java | 66 +-
.../reservation/ReservationAllocation.java | 12 +-
.../reservation/SharingPolicy.java | 24 +-
.../reservation/planning/IterativePlanner.java | 16 +-
.../reservation/planning/PlanningAlgorithm.java | 34 +-
.../reservation/planning/StageAllocator.java | 6 +-
.../planning/StageAllocatorGreedy.java | 23 +-
.../planning/StageAllocatorLowCostAligned.java | 14 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 81 +-
.../rmapp/attempt/AMLivelinessMonitor.java | 3 +-
.../rmapp/attempt/RMAppAttempt.java | 6 +
.../rmapp/attempt/RMAppAttemptImpl.java | 22 +-
.../rmcontainer/ContainerAllocationExpirer.java | 3 +-
.../server/resourcemanager/rmnode/RMNode.java | 2 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 50 +-
.../rmnode/RMNodeStatusEvent.java | 2 +-
.../scheduler/AppSchedulingInfo.java | 379 ++---
.../scheduler/ResourceUsage.java | 62 +-
.../scheduler/SchedulerApplicationAttempt.java | 124 +-
.../scheduler/SchedulerNode.java | 2 +-
.../scheduler/capacity/AbstractCSQueue.java | 8 +-
...CSAMContainerLaunchDiagnosticsConstants.java | 35 +
.../scheduler/capacity/CapacityScheduler.java | 8 +-
.../scheduler/capacity/LeafQueue.java | 41 +-
.../scheduler/capacity/QueueCapacities.java | 9 +
.../allocator/RegularContainerAllocator.java | 16 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 93 +-
.../scheduler/fair/FSAppAttempt.java | 19 +-
.../scheduler/fair/FairScheduler.java | 4 +-
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../webapp/CapacitySchedulerPage.java | 2 +-
.../webapp/MetricsOverviewTable.java | 30 +-
.../resourcemanager/webapp/NodesPage.java | 3 +
.../resourcemanager/webapp/RMWebServices.java | 384 ++++-
.../webapp/dao/AppAttemptInfo.java | 8 +
.../webapp/dao/ClusterMetricsInfo.java | 6 +
.../resourcemanager/webapp/dao/NodeInfo.java | 8 +
.../webapp/dao/ReservationDefinitionInfo.java | 82 +
.../dao/ReservationDeleteRequestInfo.java | 49 +
.../dao/ReservationDeleteResponseInfo.java | 36 +
.../webapp/dao/ReservationRequestInfo.java | 78 +
.../webapp/dao/ReservationRequestsInfo.java | 63 +
.../dao/ReservationSubmissionRequestInfo.java | 60 +
.../dao/ReservationSubmissionResponseInfo.java | 54 +
.../dao/ReservationUpdateRequestInfo.java | 60 +
.../dao/ReservationUpdateResponseInfo.java | 37 +
.../webapp/dao/ResourceUtilizationInfo.java | 89 +
.../yarn/server/resourcemanager/MockNodes.java | 2 +-
.../server/resourcemanager/TestAppManager.java | 3 +-
.../resourcemanager/TestClientRMService.java | 60 +-
.../TestLeaderElectorService.java | 269 +++
.../yarn/server/resourcemanager/TestRMHA.java | 8 +-
.../resourcemanager/TestRMNodeTransitions.java | 89 +-
.../server/resourcemanager/TestRMRestart.java | 98 +-
.../applicationsmanager/TestAMRestart.java | 8 +
.../metrics/TestSystemMetricsPublisher.java | 34 +-
.../nodelabels/NullRMNodeLabelsManager.java | 4 +-
.../recovery/RMStateStoreTestBase.java | 57 +-
.../recovery/TestFSRMStateStore.java | 17 +
.../recovery/TestLeveldbRMStateStore.java | 16 +
.../recovery/TestZKRMStateStore.java | 12 +
.../TestCapacitySchedulerPlanFollower.java | 6 +-
.../reservation/TestInMemoryPlan.java | 72 +-
.../planning/TestGreedyReservationAgent.java | 94 +-
.../rmapp/TestRMAppTransitions.java | 4 +-
.../scheduler/TestResourceUsage.java | 22 +-
.../TestSchedulerApplicationAttempt.java | 68 +-
.../capacity/TestApplicationLimits.java | 9 +-
.../TestApplicationLimitsByPartition.java | 44 +-
.../TestNodeLabelContainerAllocation.java | 17 +-
.../scheduler/capacity/TestQueueCapacities.java | 3 +-
.../scheduler/capacity/TestReservations.java | 39 +
.../scheduler/fair/TestFairScheduler.java | 98 ++
.../resourcemanager/webapp/TestNodesPage.java | 2 +-
.../resourcemanager/webapp/TestRMWebApp.java | 5 +
.../webapp/TestRMWebServices.java | 2 +-
.../webapp/TestRMWebServicesApps.java | 6 +-
.../TestRMWebServicesAppsModification.java | 10 +
.../webapp/TestRMWebServicesNodes.java | 79 +-
.../webapp/TestRMWebServicesReservation.java | 517 ++++++
.../src/test/resources/delete-reservation.json | 3 +
.../src/test/resources/submit-reservation.json | 31 +
.../src/test/resources/update-reservation.json | 31 +
.../TestMiniYarnClusterNodeUtilization.java | 2 +-
.../yarn/server/webproxy/WebAppProxyServer.java | 9 +-
.../src/site/markdown/CapacityScheduler.md | 25 +
.../src/site/markdown/NodeManager.md | 2 +-
.../src/site/markdown/TimelineServer.md | 78 +
.../src/site/markdown/YarnCommands.md | 2 +
623 files changed, 17741 insertions(+), 9099 deletions(-)
----------------------------------------------------------------------