You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/05/09 17:36:59 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-1312
Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a65c707c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a65c707c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a65c707c
Branch: refs/heads/HDFS-1312
Commit: a65c707c0f6239112b2613a9e1ecbd01d963c03a
Parents: 4f38f88 411fb4b
Author: Anu Engineer <ae...@apache.org>
Authored: Mon May 9 10:35:42 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon May 9 10:35:42 2016 -0700
----------------------------------------------------------------------
LICENSE.txt | 269 ++++++
NOTICE.txt | 15 +
dev-support/bin/create-release | 623 +++++++++++++
dev-support/bin/dist-copynativelibs | 164 ++++
dev-support/bin/dist-layout-stitching | 2 +
dev-support/bin/yetus-wrapper | 2 +-
dev-support/create-release.sh | 144 ---
dev-support/docker/Dockerfile | 86 +-
dev-support/verify-xml.sh | 150 ++++
.../main/resources/assemblies/hadoop-dist.xml | 2 +-
.../main/resources/assemblies/hadoop-tools.xml | 7 +
.../client/KerberosAuthenticator.java | 6 +-
.../hadoop-common/src/main/bin/hadoop | 10 +
.../hadoop-common/src/main/bin/hadoop-config.sh | 6 +-
.../src/main/bin/hadoop-functions.sh | 37 +
.../src/main/conf/hadoop-metrics.properties | 75 --
.../apache/hadoop/fs/AbstractFileSystem.java | 11 +
.../apache/hadoop/fs/ChecksumFileSystem.java | 130 ++-
.../hadoop/fs/CommonConfigurationKeys.java | 24 +-
.../java/org/apache/hadoop/fs/CreateFlag.java | 9 +-
.../java/org/apache/hadoop/fs/FileContext.java | 17 +
.../java/org/apache/hadoop/fs/FileSystem.java | 7 +-
.../java/org/apache/hadoop/fs/FilterFs.java | 6 +
.../apache/hadoop/fs/TrashPolicyDefault.java | 1 -
.../apache/hadoop/fs/WindowsGetSpaceUsed.java | 3 +-
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 +
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 8 +
.../org/apache/hadoop/http/HttpServer2.java | 4 -
.../hadoop/io/compress/DecompressorStream.java | 41 +-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 114 ++-
.../io/erasurecode/ErasureCodeConstants.java | 40 +
.../erasurecode/coder/HHXORErasureEncoder.java | 2 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 1 +
.../io/erasurecode/coder/RSErasureEncoder.java | 3 +-
.../hadoop/io/file/tfile/Compression.java | 6 +-
.../ipc/AsyncCallLimitExceededException.java | 36 +
.../org/apache/hadoop/ipc/CallQueueManager.java | 6 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 80 +-
.../apache/hadoop/ipc/DecayRpcScheduler.java | 126 ++-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 34 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 15 +-
.../java/org/apache/hadoop/log/LogLevel.java | 16 +-
.../apache/hadoop/metrics/ContextFactory.java | 214 -----
.../apache/hadoop/metrics/MetricsContext.java | 125 ---
.../apache/hadoop/metrics/MetricsException.java | 49 --
.../apache/hadoop/metrics/MetricsRecord.java | 254 ------
.../apache/hadoop/metrics/MetricsServlet.java | 188 ----
.../org/apache/hadoop/metrics/MetricsUtil.java | 105 ---
.../java/org/apache/hadoop/metrics/Updater.java | 41 -
.../org/apache/hadoop/metrics/file/package.html | 43 -
.../hadoop/metrics/ganglia/GangliaContext.java | 276 ------
.../metrics/ganglia/GangliaContext31.java | 147 ----
.../apache/hadoop/metrics/ganglia/package.html | 80 --
.../apache/hadoop/metrics/jvm/EventCounter.java | 36 -
.../apache/hadoop/metrics/jvm/JvmMetrics.java | 203 -----
.../apache/hadoop/metrics/jvm/package-info.java | 22 -
.../java/org/apache/hadoop/metrics/package.html | 159 ----
.../metrics/spi/AbstractMetricsContext.java | 494 -----------
.../hadoop/metrics/spi/CompositeContext.java | 206 -----
.../apache/hadoop/metrics/spi/MetricValue.java | 58 --
.../hadoop/metrics/spi/MetricsRecordImpl.java | 304 -------
.../metrics/spi/NoEmitMetricsContext.java | 61 --
.../apache/hadoop/metrics/spi/NullContext.java | 74 --
.../spi/NullContextWithUpdateThread.java | 82 --
.../apache/hadoop/metrics/spi/OutputRecord.java | 93 --
.../org/apache/hadoop/metrics/spi/Util.java | 68 --
.../org/apache/hadoop/metrics/spi/package.html | 36 -
.../apache/hadoop/metrics/util/MBeanUtil.java | 92 --
.../apache/hadoop/metrics/util/MetricsBase.java | 51 --
.../metrics/util/MetricsDynamicMBeanBase.java | 229 -----
.../hadoop/metrics/util/MetricsIntValue.java | 106 ---
.../hadoop/metrics/util/MetricsLongValue.java | 93 --
.../hadoop/metrics/util/MetricsRegistry.java | 90 --
.../metrics/util/MetricsTimeVaryingInt.java | 129 ---
.../metrics/util/MetricsTimeVaryingLong.java | 125 ---
.../metrics/util/MetricsTimeVaryingRate.java | 198 -----
.../hadoop/metrics/util/package-info.java | 22 -
.../org/apache/hadoop/metrics2/lib/Interns.java | 12 +-
.../hadoop/metrics2/source/JvmMetrics.java | 4 +
.../hadoop/metrics2/util/Metrics2Util.java | 105 +++
.../org/apache/hadoop/net/NetworkTopology.java | 109 ++-
.../org/apache/hadoop/security/Credentials.java | 169 +++-
.../hadoop/security/LdapGroupsMapping.java | 41 +-
.../hadoop/security/UserGroupInformation.java | 17 +-
.../apache/hadoop/security/token/DtFetcher.java | 41 +
.../hadoop/security/token/DtFileOperations.java | 271 ++++++
.../hadoop/security/token/DtUtilShell.java | 326 +++++++
.../org/apache/hadoop/security/token/Token.java | 94 +-
.../org/apache/hadoop/tools/CommandShell.java | 114 +++
.../hadoop/util/GenericOptionsParser.java | 8 +-
.../java/org/apache/hadoop/util/RunJar.java | 99 ++-
.../main/java/org/apache/hadoop/util/Shell.java | 23 +-
.../org/apache/hadoop/util/SysInfoWindows.java | 2 +-
.../hadoop-common/src/main/proto/Security.proto | 11 +
.../src/main/resources/core-default.xml | 31 +-
.../src/site/markdown/ClusterSetup.md | 2 +-
.../src/site/markdown/CommandsManual.md | 25 +-
.../src/site/markdown/CredentialProviderAPI.md | 35 +
.../src/site/markdown/DeprecatedProperties.md | 3 +-
.../src/site/markdown/SecureMode.md | 8 +-
.../src/site/markdown/SingleCluster.md.vm | 2 +-
.../hadoop-common/src/site/markdown/Tracing.md | 2 +-
.../src/site/markdown/UnixShellGuide.md | 32 +-
.../src/site/markdown/filesystem/filesystem.md | 2 +-
.../conf/TestCommonConfigurationFields.java | 176 ++++
.../conf/TestConfigurationFieldsBase.java | 255 ++++++
.../kms/TestLoadBalancingKMSClientProvider.java | 8 +-
.../hadoop/fs/TestChecksumFileSystem.java | 19 +
.../hadoop/fs/TestDelegateToFileSystem.java | 2 +-
.../apache/hadoop/fs/shell/find/TestAnd.java | 25 +-
.../fs/shell/find/TestFilterExpression.java | 27 +-
.../apache/hadoop/fs/shell/find/TestFind.java | 3 +-
.../apache/hadoop/fs/shell/find/TestIname.java | 17 +-
.../apache/hadoop/fs/shell/find/TestName.java | 17 +-
.../apache/hadoop/fs/shell/find/TestPrint.java | 9 +-
.../apache/hadoop/fs/shell/find/TestPrint0.java | 9 +-
.../apache/hadoop/fs/shell/find/TestResult.java | 41 +-
.../apache/hadoop/ha/TestSshFenceByTcpPort.java | 2 +-
.../org/apache/hadoop/http/TestHttpServer.java | 10 +-
.../hadoop/io/compress/FakeCompressor.java | 113 +++
.../hadoop/io/compress/FakeDecompressor.java | 109 +++
.../compress/TestBlockDecompressorStream.java | 180 +---
.../io/compress/TestDecompressorStream.java | 99 +++
.../erasurecode/TestCodecRawCoderMapping.java | 85 ++
.../hadoop/io/erasurecode/TestCoderBase.java | 2 +-
.../coder/TestHHXORErasureCoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 2 +-
.../hadoop/io/file/tfile/TestCompression.java | 41 +
.../org/apache/hadoop/ipc/TestAsyncIPC.java | 201 ++++-
.../java/org/apache/hadoop/ipc/TestRPC.java | 90 +-
.../org/apache/hadoop/log/TestLogLevel.java | 39 +-
.../hadoop/metrics/TestMetricsServlet.java | 112 ---
.../metrics/ganglia/TestGangliaContext.java | 84 --
.../hadoop/metrics/spi/TestOutputRecord.java | 39 -
.../apache/hadoop/security/TestCredentials.java | 246 +++++-
.../hadoop/security/TestLdapGroupsMapping.java | 2 +-
.../hadoop/security/token/TestDtFetcher.java | 41 +
.../hadoop/security/token/TestDtUtilShell.java | 264 ++++++
.../apache/hadoop/tools/TestCommandShell.java | 128 +++
.../java/org/apache/hadoop/util/TestRunJar.java | 69 +-
.../java/org/apache/hadoop/util/TestShell.java | 37 +
.../org.apache.hadoop.security.token.DtFetcher | 14 +
.../src/test/scripts/hadoop_confdir.bats | 7 +-
.../src/test/scripts/hadoop_escape_chars.bats | 32 +
.../hadoop-kms/src/main/conf/kms-env.sh | 2 +-
.../hadoop-kms/src/main/libexec/kms-config.sh | 2 +-
.../hadoop-kms/src/main/sbin/kms.sh | 8 +-
.../hadoop-kms/src/site/markdown/index.md.vm | 4 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +
.../main/java/org/apache/hadoop/fs/Hdfs.java | 5 +
.../org/apache/hadoop/hdfs/AddBlockFlag.java | 59 ++
.../hadoop/hdfs/AsyncDistributedFileSystem.java | 114 +++
.../apache/hadoop/hdfs/BlockReaderFactory.java | 870 ------------------
.../apache/hadoop/hdfs/BlockReaderLocal.java | 724 ---------------
.../hadoop/hdfs/BlockReaderLocalLegacy.java | 744 ----------------
.../org/apache/hadoop/hdfs/BlockReaderUtil.java | 57 --
.../org/apache/hadoop/hdfs/DFSInputStream.java | 1 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 27 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 2 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 8 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 16 +-
.../hadoop/hdfs/DistributedFileSystem.java | 25 +-
.../apache/hadoop/hdfs/ExternalBlockReader.java | 132 ---
.../apache/hadoop/hdfs/HdfsConfiguration.java | 4 +-
.../apache/hadoop/hdfs/RemoteBlockReader.java | 510 -----------
.../apache/hadoop/hdfs/RemoteBlockReader2.java | 472 ----------
.../apache/hadoop/hdfs/StripedDataStreamer.java | 6 +-
.../hdfs/client/HdfsClientConfigKeys.java | 10 +-
.../hdfs/client/impl/BlockReaderFactory.java | 878 +++++++++++++++++++
.../hdfs/client/impl/BlockReaderLocal.java | 725 +++++++++++++++
.../client/impl/BlockReaderLocalLegacy.java | 745 ++++++++++++++++
.../hdfs/client/impl/BlockReaderRemote.java | 512 +++++++++++
.../hdfs/client/impl/BlockReaderRemote2.java | 474 ++++++++++
.../hdfs/client/impl/BlockReaderUtil.java | 58 ++
.../hdfs/client/impl/ExternalBlockReader.java | 134 +++
.../hadoop/hdfs/protocol/ClientProtocol.java | 5 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 4 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 5 +-
.../ClientNamenodeProtocolTranslatorPB.java | 53 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 27 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 6 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 5 +
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 8 +-
.../hadoop-hdfs-native-client/pom.xml | 5 +-
.../src/CMakeLists.txt | 12 +-
.../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 6 +-
.../dev-support/findbugsExcludeFile.xml | 2 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 31 +-
.../org/apache/hadoop/hdfs/HdfsDtFetcher.java | 82 ++
.../apache/hadoop/hdfs/SWebHdfsDtFetcher.java | 39 +
.../apache/hadoop/hdfs/WebHdfsDtFetcher.java | 39 +
.../hdfs/client/CreateEncryptionZoneFlag.java | 71 ++
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 89 +-
.../apache/hadoop/hdfs/client/package-info.java | 27 +
.../hadoop/hdfs/protocol/BlockListAsLongs.java | 93 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 6 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 11 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 27 +-
.../AvailableSpaceBlockPlacementPolicy.java | 11 +-
.../server/blockmanagement/BlockManager.java | 205 +++--
.../blockmanagement/BlockManagerSafeMode.java | 4 +-
.../blockmanagement/BlockPlacementPolicy.java | 11 +-
.../BlockPlacementPolicyDefault.java | 138 +--
.../blockmanagement/CorruptReplicasMap.java | 12 +
.../blockmanagement/DecommissionManager.java | 2 +-
.../blockmanagement/ErasureCodingWork.java | 2 +-
.../blockmanagement/ExcessRedundancyMap.java | 114 +++
.../blockmanagement/ExcessReplicaMap.java | 106 ---
.../PendingReconstructionBlocks.java | 287 ++++++
.../PendingReplicationBlocks.java | 291 ------
.../server/blockmanagement/ReplicationWork.java | 3 +-
.../hdfs/server/datanode/DirectoryScanner.java | 14 +-
.../hdfs/server/datanode/VolumeScanner.java | 2 +-
.../erasurecode/StripedBlockReader.java | 4 +-
.../erasurecode/StripedReconstructor.java | 2 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 10 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 7 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 19 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 4 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 4 +-
.../namenode/ErasureCodingPolicyManager.java | 14 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 9 +-
.../hdfs/server/namenode/FSEditLogOp.java | 3 +-
.../server/namenode/FSImageFormatProtobuf.java | 17 +-
.../hdfs/server/namenode/FSNamesystem.java | 20 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 155 ++--
.../hdfs/server/namenode/NameNodeRpcServer.java | 17 +-
.../hdfs/server/namenode/NamenodeFsck.java | 2 +-
.../ha/RequestHedgingProxyProvider.java | 2 +-
.../top/window/RollingWindowManager.java | 73 +-
.../web/resources/NamenodeWebHdfsMethods.java | 13 +-
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 69 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 4 +
.../hdfs/tools/DelegationTokenFetcher.java | 3 +-
.../org/apache/hadoop/hdfs/tools/JMXGet.java | 2 +-
.../StatisticsEditsVisitor.java | 11 +-
.../org.apache.hadoop.security.token.DtFetcher | 18 +
.../src/main/resources/hdfs-default.xml | 29 +-
.../src/site/markdown/HDFSCommands.md | 3 +
.../src/site/markdown/HDFSErasureCoding.md | 6 +-
.../markdown/HDFSHighAvailabilityWithNFS.md | 12 +-
.../markdown/HDFSHighAvailabilityWithQJM.md | 14 +-
.../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 2 +-
.../src/site/markdown/HdfsUserGuide.md | 2 +-
.../src/site/markdown/TransparentEncryption.md | 13 +-
.../hadoop-hdfs/src/site/markdown/ViewFs.md | 16 +-
.../hadoop/fs/TestEnhancedByteBufferAccess.java | 3 +-
.../org/apache/hadoop/fs/TestGlobPaths.java | 19 +
.../apache/hadoop/hdfs/BlockReaderTestUtil.java | 258 ------
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 14 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 25 +-
.../hadoop/hdfs/TestAppendSnapshotTruncate.java | 2 +-
.../apache/hadoop/hdfs/TestAsyncDFSRename.java | 340 +++++++
.../apache/hadoop/hdfs/TestBlockReaderBase.java | 94 --
.../hadoop/hdfs/TestBlockReaderFactory.java | 534 -----------
.../hadoop/hdfs/TestBlockReaderLocal.java | 780 ----------------
.../hadoop/hdfs/TestBlockReaderLocalLegacy.java | 220 -----
.../hadoop/hdfs/TestBlockStoragePolicy.java | 6 +-
.../hdfs/TestClientBlockVerification.java | 125 ---
.../org/apache/hadoop/hdfs/TestConnCache.java | 1 +
.../hadoop/hdfs/TestDFSClientFailover.java | 2 +-
.../hadoop/hdfs/TestDFSClientRetries.java | 12 +-
.../apache/hadoop/hdfs/TestDFSOutputStream.java | 62 +-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 2 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 36 +-
.../apache/hadoop/hdfs/TestDatanodeDeath.java | 4 +-
.../apache/hadoop/hdfs/TestDecommission.java | 2 +-
.../hdfs/TestDecommissionWithStriped.java | 2 +-
.../hadoop/hdfs/TestDisableConnCache.java | 1 +
.../apache/hadoop/hdfs/TestEncryptionZones.java | 238 ++++-
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 7 +-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 4 +-
.../org/apache/hadoop/hdfs/TestFileAppend2.java | 2 +-
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 2 +-
.../apache/hadoop/hdfs/TestFileCorruption.java | 87 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 4 +-
.../org/apache/hadoop/hdfs/TestGetBlocks.java | 8 +-
.../java/org/apache/hadoop/hdfs/TestHFlush.java | 10 +-
.../hadoop/hdfs/TestParallelReadUtil.java | 1 +
.../java/org/apache/hadoop/hdfs/TestQuota.java | 2 +-
.../hadoop/hdfs/TestRemoteBlockReader.java | 29 -
.../hadoop/hdfs/TestRemoteBlockReader2.java | 25 -
.../hdfs/TestReplaceDatanodeOnFailure.java | 2 +-
.../org/apache/hadoop/hdfs/TestReplication.java | 6 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 12 +-
.../hadoop/hdfs/TestSetrepIncreasing.java | 2 +-
.../hdfs/client/impl/BlockReaderTestUtil.java | 267 ++++++
.../hdfs/client/impl/TestBlockReaderBase.java | 97 ++
.../client/impl/TestBlockReaderFactory.java | 539 ++++++++++++
.../hdfs/client/impl/TestBlockReaderLocal.java | 786 +++++++++++++++++
.../client/impl/TestBlockReaderLocalLegacy.java | 227 +++++
.../hdfs/client/impl/TestBlockReaderRemote.java | 30 +
.../client/impl/TestBlockReaderRemote2.java | 27 +
.../impl/TestClientBlockVerification.java | 126 +++
.../BaseReplicationPolicyTest.java | 2 +-
.../TestAvailableSpaceBlockPlacementPolicy.java | 2 +-
.../blockmanagement/TestBlockManager.java | 77 +-
.../TestBlockManagerSafeMode.java | 64 +-
.../blockmanagement/TestBlockTokenWithDFS.java | 2 +-
.../TestBlocksWithNotEnoughRacks.java | 2 +-
.../TestOverReplicatedBlocks.java | 2 +-
.../TestPendingDataNodeMessages.java | 4 +-
.../TestPendingReconstruction.java | 418 +++++++++
.../blockmanagement/TestPendingReplication.java | 418 ---------
.../blockmanagement/TestReplicationPolicy.java | 47 +-
.../TestReplicationPolicyConsiderLoad.java | 4 +-
.../TestReplicationPolicyWithNodeGroup.java | 8 +-
.../TestReplicationPolicyWithUpgradeDomain.java | 2 +-
.../server/datanode/FsDatasetTestUtils.java | 7 +
.../server/datanode/TestBlockPoolManager.java | 12 +-
.../hdfs/server/datanode/TestBlockScanner.java | 63 ++
.../server/datanode/TestDataNodeMetrics.java | 6 +-
.../datanode/TestDataNodeVolumeFailure.java | 2 +-
.../server/datanode/TestDirectoryScanner.java | 2 +-
.../server/datanode/TestFsDatasetCache.java | 3 +-
.../datanode/TestFsDatasetCacheRevocation.java | 2 +-
.../server/datanode/TestLargeBlockReport.java | 143 +++
.../fsdataset/impl/FsDatasetImplTestUtils.java | 32 +
.../fsdataset/impl/TestFsDatasetImpl.java | 15 +-
.../fsdataset/impl/TestFsVolumeList.java | 36 +-
.../server/namenode/NNThroughputBenchmark.java | 2 +-
.../hdfs/server/namenode/TestAddBlockRetry.java | 8 +-
.../server/namenode/TestAddStripedBlocks.java | 2 +-
.../hdfs/server/namenode/TestAllowFormat.java | 4 +-
...stBlockPlacementPolicyRackFaultTolerant.java | 4 +-
.../server/namenode/TestCacheDirectives.java | 2 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../namenode/TestDecommissioningStatus.java | 4 +-
.../TestDefaultBlockPlacementPolicy.java | 2 +-
.../hdfs/server/namenode/TestDeleteRace.java | 7 +-
.../namenode/TestFSImageWithSnapshot.java | 43 +-
.../hdfs/server/namenode/TestFileTruncate.java | 2 +-
.../hdfs/server/namenode/TestHostsFiles.java | 2 +-
.../hdfs/server/namenode/TestMetaSave.java | 6 +-
.../namenode/TestNameNodeReconfigure.java | 81 +-
.../server/namenode/TestNamenodeRetryCache.java | 4 +-
.../namenode/TestProcessCorruptBlocks.java | 8 +-
.../namenode/TestReconstructStripedBlocks.java | 6 +-
.../TestUpgradeDomainBlockPlacementPolicy.java | 2 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 2 +-
.../hdfs/server/namenode/ha/TestHASafeMode.java | 2 +-
.../ha/TestPendingCorruptDnMessages.java | 51 +-
.../ha/TestRequestHedgingProxyProvider.java | 6 +-
.../namenode/ha/TestRetryCacheWithHA.java | 4 +-
.../snapshot/TestOpenFilesWithSnapshot.java | 2 +-
.../snapshot/TestSnapshotBlocksMap.java | 6 +-
.../shortcircuit/TestShortCircuitCache.java | 4 +-
.../shortcircuit/TestShortCircuitLocalRead.java | 16 +-
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +-
.../TestOfflineEditsViewer.java | 30 +
.../hadoop/hdfs/web/TestWebHdfsTimeouts.java | 25 +-
.../apache/hadoop/net/TestNetworkTopology.java | 75 +-
.../org/apache/hadoop/tools/TestJMXGet.java | 16 +-
.../src/test/resources/hdfs-site.malformed.xml | 16 +-
.../src/test/resources/testCryptoConf.xml | 86 ++
.../test/resources/testErasureCodingConf.xml | 12 +-
.../jobhistory/JobHistoryEventHandler.java | 9 +-
.../v2/app/job/event/TaskAttemptKillEvent.java | 15 +-
.../app/job/event/TaskTAttemptKilledEvent.java | 40 +
.../mapreduce/v2/app/job/impl/JobImpl.java | 4 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 48 +-
.../mapreduce/v2/app/job/impl/TaskImpl.java | 25 +-
.../v2/app/local/LocalContainerAllocator.java | 2 +
.../v2/app/rm/RMContainerAllocator.java | 66 +-
.../v2/app/rm/RMContainerRequestor.java | 6 +
.../hadoop/mapreduce/v2/app/TestMRApp.java | 51 +-
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 3 +-
.../v2/app/job/impl/TestTaskAttempt.java | 87 +-
.../mapreduce/v2/app/job/impl/TestTaskImpl.java | 75 +-
.../app/local/TestLocalContainerAllocator.java | 43 +
.../v2/app/rm/TestRMContainerAllocator.java | 250 ++++++
.../apache/hadoop/mapred/LocalJobRunner.java | 2 +-
.../hadoop/mapred/LocalJobRunnerMetrics.java | 94 +-
.../hadoop/mapreduce/task/reduce/Shuffle.java | 2 +-
.../task/reduce/ShuffleClientMetrics.java | 91 +-
.../hadoop/mapred/TestFileInputFormat.java | 2 +-
.../lib/input/TestFileInputFormat.java | 2 +-
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 9 +-
.../mapreduce/v2/hs/HistoryFileManager.java | 19 +-
.../mapreduce/v2/hs/TestJobHistoryEntities.java | 27 +
.../src/test/resources/job_0.23.9-FAILED.jhist | 2 +-
.../resources/job_1329348432655_0001_conf.xml | 14 +-
.../java/org/apache/hadoop/mapred/MRBench.java | 18 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 9 -
.../hadoop/examples/terasort/TestTeraSort.java | 8 +-
hadoop-project-dist/pom.xml | 131 +--
hadoop-project/src/site/site.xml | 1 +
.../java/org/apache/hadoop/ant/DfsTask.java | 9 +-
.../apache/hadoop/ant/condition/DfsExists.java | 2 +-
.../apache/hadoop/ant/condition/DfsIsDir.java | 2 +-
.../apache/hadoop/ant/condition/DfsZeroLen.java | 2 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 10 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 12 +
.../src/site/markdown/tools/hadoop-aws/index.md | 11 +-
.../hadoop/fs/s3a/TestS3AConfiguration.java | 47 +-
.../dev-support/findbugs-exclude.xml | 29 +
hadoop-tools/hadoop-datajoin/pom.xml | 11 +
.../org/apache/hadoop/tools/CopyListing.java | 4 +
.../java/org/apache/hadoop/tools/DistCp.java | 3 +-
.../org/apache/hadoop/tools/DistCpSync.java | 26 +
.../apache/hadoop/tools/SimpleCopyListing.java | 10 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 6 +-
.../src/site/markdown/DistCp.md.vm | 72 +-
.../org/apache/hadoop/tools/TestDistCpSync.java | 65 ++
.../apache/hadoop/tools/TestDistCpSystem.java | 34 +
.../apache/hadoop/tools/TestOptionsParser.java | 286 +++---
.../hadoop/tools/mapred/TestCopyMapper.java | 85 ++
.../src/site/markdown/GridMix.md.vm | 18 +-
.../hadoop-openstack/src/site/markdown/index.md | 2 +-
.../src/main/data/2jobs2min-rumen-jh.json | 48 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 14 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 14 +
.../api/records/ContainerLaunchContext.java | 33 +
.../yarn/api/records/ContainerRetryContext.java | 84 ++
.../yarn/api/records/ContainerRetryPolicy.java | 35 +
.../hadoop/yarn/api/records/ContainerState.java | 5 +-
.../yarn/api/records/ResourceRequest.java | 41 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 95 ++
.../src/main/proto/yarn_protos.proto | 16 +
.../yarn/conf/TestYarnConfigurationFields.java | 25 +
.../pom.xml | 6 +
.../distributedshell/ApplicationMaster.java | 53 +-
.../applications/distributedshell/Client.java | 34 +
.../distributedshell/TestDistributedShell.java | 50 ++
.../hadoop/yarn/client/api/AMRMClient.java | 44 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 506 +++++++----
.../yarn/client/api/impl/TestYarnClient.java | 9 +-
.../hadoop/yarn/client/cli/TestLogsCLI.java | 215 ++++-
.../impl/pb/ContainerLaunchContextPBImpl.java | 56 +-
.../impl/pb/ContainerRetryContextPBImpl.java | 177 ++++
.../yarn/api/records/impl/pb/ProtoUtils.java | 30 +
.../records/impl/pb/ResourceRequestPBImpl.java | 21 +
.../client/api/impl/TimelineClientImpl.java | 74 +-
.../hadoop/yarn/event/EventDispatcher.java | 137 +++
.../logaggregation/AggregatedLogFormat.java | 20 +
.../yarn/logaggregation/LogCLIHelpers.java | 231 ++++-
.../yarn/util/ProcfsBasedProcessTree.java | 19 +-
.../src/main/resources/yarn-default.xml | 43 +
.../hadoop/yarn/api/TestPBImplRecords.java | 12 +-
.../TestApplicationClientProtocolRecords.java | 69 ++
.../client/api/impl/TestTimelineClient.java | 43 +-
.../yarn/util/TestProcfsBasedProcessTree.java | 39 +-
.../hadoop/yarn/webapp/JerseyTestBase.java | 18 +-
.../registry/client/types/AddressTypes.java | 2 +-
.../hadoop/registry/RegistryTestHelper.java | 4 +-
...pplicationHistoryManagerOnTimelineStore.java | 21 +-
.../webapp/AHSWebServices.java | 270 +++++-
...pplicationHistoryManagerOnTimelineStore.java | 29 +-
.../webapp/TestAHSWebServices.java | 203 ++++-
.../hadoop-yarn-server-common/pom.xml | 1 +
.../api/DistributedSchedulerProtocol.java | 78 ++
.../api/DistributedSchedulerProtocolPB.java | 36 +
.../hadoop/yarn/server/api/ServerRMProxy.java | 4 +
...istributedSchedulerProtocolPBClientImpl.java | 151 ++++
...stributedSchedulerProtocolPBServiceImpl.java | 143 +++
.../DistSchedAllocateResponse.java | 58 ++
.../DistSchedRegisterResponse.java | 102 +++
.../pb/DistSchedAllocateResponsePBImpl.java | 180 ++++
.../pb/DistSchedRegisterResponsePBImpl.java | 304 +++++++
.../yarn/server/api/records/NodeStatus.java | 9 +
.../api/records/QueuedContainersStatus.java | 45 +
.../api/records/impl/pb/NodeStatusPBImpl.java | 40 +-
.../impl/pb/QueuedContainersStatusPBImpl.java | 80 ++
.../hadoop/yarn/server/utils/BuilderUtils.java | 9 +
.../yarn/server/webapp/dao/ContainerInfo.java | 6 +
.../proto/distributed_scheduler_protocol.proto | 38 +
.../main/proto/yarn_server_common_protos.proto | 6 +
.../yarn_server_common_service_protos.proto | 15 +
.../protocolrecords/TestProtocolRecords.java | 30 +
.../server/nodemanager/ContainerExecutor.java | 1 +
.../hadoop/yarn/server/nodemanager/Context.java | 22 +
.../nodemanager/DefaultContainerExecutor.java | 4 +-
.../nodemanager/DockerContainerExecutor.java | 4 +-
.../nodemanager/LinuxContainerExecutor.java | 2 +-
.../nodemanager/LocalDirsHandlerService.java | 22 +
.../yarn/server/nodemanager/NodeManager.java | 77 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 8 +
.../nodemanager/amrmproxy/AMRMProxyService.java | 8 +
.../amrmproxy/AbstractRequestInterceptor.java | 42 +
.../amrmproxy/DefaultRequestInterceptor.java | 131 ++-
.../amrmproxy/RequestInterceptor.java | 4 +-
.../containermanager/ContainerManagerImpl.java | 133 +--
.../containermanager/container/Container.java | 12 +
.../container/ContainerImpl.java | 188 +++-
.../container/ContainerState.java | 6 +-
.../launcher/ContainerLaunch.java | 256 ++++--
.../launcher/ContainerRelaunch.java | 196 +++++
.../launcher/ContainersLauncher.java | 10 +
.../launcher/ContainersLauncherEventType.java | 1 +
.../privileged/PrivilegedOperationExecutor.java | 8 +-
.../runtime/DefaultLinuxContainerRuntime.java | 4 +-
.../runtime/DockerLinuxContainerRuntime.java | 51 +-
.../logaggregation/LogAggregationService.java | 7 +-
.../monitor/ContainerMetrics.java | 15 +-
.../monitor/ContainersMonitor.java | 18 +
.../monitor/ContainersMonitorImpl.java | 160 +++-
.../queuing/QueuingContainerManagerImpl.java | 556 ++++++++++++
.../containermanager/queuing/package-info.java | 23 +
.../recovery/NMLeveldbStateStoreService.java | 172 +++-
.../recovery/NMNullStateStoreService.java | 19 +-
.../recovery/NMStateStoreService.java | 71 +-
.../nodemanager/scheduler/LocalScheduler.java | 416 +++++++++
.../OpportunisticContainerAllocator.java | 185 ++++
.../security/NMTokenSecretManagerInNM.java | 24 +-
.../nodemanager/webapp/NMWebServices.java | 22 +-
.../yarn/server/nodemanager/TestEventFlow.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 5 +-
.../amrmproxy/BaseAMRMProxyTest.java | 17 +-
.../impl/pb/TestPBRecordImpl.java | 2 +-
.../BaseContainerManagerTest.java | 58 +-
.../containermanager/TestContainerManager.java | 78 +-
.../TestContainerManagerRecovery.java | 6 +-
.../container/TestContainer.java | 85 +-
.../launcher/TestContainerLaunch.java | 2 +-
.../runtime/TestDockerContainerRuntime.java | 117 ++-
.../TestLocalCacheDirectoryManager.java | 3 +-
.../TestResourceLocalizationService.java | 4 +-
.../TestLogAggregationService.java | 11 +-
.../monitor/MockResourceCalculatorPlugin.java | 4 +
.../monitor/TestContainerMetrics.java | 7 +-
.../TestContainersMonitorResourceChange.java | 9 +-
.../queuing/TestQueuingContainerManager.java | 301 +++++++
.../recovery/NMMemoryStateStoreService.java | 34 +-
.../TestNMLeveldbStateStoreService.java | 47 +-
.../scheduler/TestLocalScheduler.java | 212 +++++
.../nodemanager/webapp/MockContainer.java | 28 +
.../webapp/TestContainerLogsPage.java | 6 +-
.../nodemanager/webapp/TestNMAppsPage.java | 3 +-
.../nodemanager/webapp/TestNMWebServer.java | 4 +-
.../nodemanager/webapp/TestNMWebServices.java | 14 +-
.../webapp/TestNMWebServicesApps.java | 2 +-
.../webapp/TestNMWebServicesContainers.java | 2 +-
.../ApplicationMasterService.java | 39 +-
.../server/resourcemanager/ClientRMService.java | 10 +-
.../server/resourcemanager/ClusterMonitor.java | 36 +
.../DistributedSchedulingService.java | 295 +++++++
.../resourcemanager/NodesListManager.java | 156 +++-
.../server/resourcemanager/RMAuditLogger.java | 47 +-
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../server/resourcemanager/ResourceManager.java | 158 +---
.../metrics/ContainerFinishedEvent.java | 9 +-
.../metrics/SystemMetricsPublisher.java | 8 +-
.../monitor/SchedulingMonitor.java | 16 +-
.../CapacitySchedulerPreemptionUtils.java | 35 +-
.../capacity/FifoCandidatesSelector.java | 26 +-
.../capacity/PreemptableResourceCalculator.java | 139 ++-
.../ProportionalCapacityPreemptionPolicy.java | 119 +--
.../ReservedContainerCandidatesSelector.java | 316 +++++++
.../monitor/capacity/TempQueuePerPartition.java | 174 +++-
.../recovery/LeveldbRMStateStore.java | 42 +
.../rmcontainer/RMContainer.java | 2 +
.../rmcontainer/RMContainerImpl.java | 15 +-
.../server/resourcemanager/rmnode/RMNode.java | 7 +
.../resourcemanager/rmnode/RMNodeImpl.java | 54 +-
.../rmnode/RMNodeStatusEvent.java | 7 +
.../scheduler/AbstractYarnScheduler.java | 6 +-
.../scheduler/AppSchedulingInfo.java | 3 +-
.../scheduler/SchedulerApplicationAttempt.java | 4 +
.../scheduler/SchedulerNode.java | 65 +-
.../scheduler/capacity/CapacityScheduler.java | 10 +-
.../CapacitySchedulerConfiguration.java | 24 +-
.../scheduler/capacity/LeafQueue.java | 7 -
.../scheduler/common/fica/FiCaSchedulerApp.java | 9 +-
.../common/fica/FiCaSchedulerNode.java | 7 +-
.../scheduler/distributed/TopKNodeSelector.java | 223 +++++
.../fair/AllocationFileLoaderService.java | 6 +
.../scheduler/fair/FSAppAttempt.java | 19 +-
.../scheduler/fair/FairScheduler.java | 19 +-
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../security/DelegationTokenRenewer.java | 9 +-
.../security/QueueACLsManager.java | 32 +-
.../resourcemanager/webapp/RMWebServices.java | 30 +-
.../yarn/server/resourcemanager/MockAM.java | 33 +-
.../yarn/server/resourcemanager/MockNodes.java | 15 +-
.../yarn/server/resourcemanager/MockRM.java | 337 ++++---
.../resourcemanager/TestApplicationACLs.java | 5 +-
.../resourcemanager/TestApplicationCleanup.java | 9 +-
.../TestApplicationMasterLauncher.java | 11 +-
.../TestApplicationMasterService.java | 5 +-
.../resourcemanager/TestClientRMService.java | 68 +-
.../TestContainerResourceUsage.java | 6 +-
.../TestDistributedSchedulingService.java | 175 ++++
.../yarn/server/resourcemanager/TestRM.java | 12 +-
.../resourcemanager/TestRMAuditLogger.java | 44 +-
.../resourcemanager/TestRMDispatcher.java | 6 +-
.../server/resourcemanager/TestRMHAMetrics.java | 76 ++
.../resourcemanager/TestRMNodeTransitions.java | 4 +-
.../server/resourcemanager/TestRMRestart.java | 35 +-
.../TestResourceTrackerService.java | 487 +++++++++-
.../resourcemanager/TestSignalContainer.java | 2 +-
.../TestWorkPreservingRMRestart.java | 72 +-
.../ahs/TestRMApplicationHistoryWriter.java | 7 +-
.../TestAMRMRPCNodeUpdates.java | 8 +-
.../applicationsmanager/TestAMRestart.java | 73 +-
.../metrics/TestSystemMetricsPublisher.java | 30 +
...alCapacityPreemptionPolicyMockFramework.java | 689 +++++++++++++++
...estProportionalCapacityPreemptionPolicy.java | 62 +-
...pacityPreemptionPolicyForNodePartitions.java | 687 +--------------
...tyPreemptionPolicyForReservedContainers.java | 430 +++++++++
...alCapacityPreemptionPolicyMockFramework.java | 247 ++++++
.../recovery/TestLeveldbRMStateStore.java | 23 +
.../rmapp/TestNodesListManager.java | 2 +-
.../scheduler/TestAbstractYarnScheduler.java | 5 +-
.../TestSchedulerApplicationAttempt.java | 3 +-
.../scheduler/TestSchedulerUtils.java | 23 +
.../CapacitySchedulerPreemptionTestBase.java | 149 ++++
.../capacity/TestCapacityScheduler.java | 28 +-
.../TestCapacitySchedulerLazyPreemption.java | 638 ++++++++++++++
.../TestCapacitySchedulerNodeLabelUpdate.java | 12 +-
.../TestCapacitySchedulerPreemption.java | 683 ---------------
...TestCapacitySchedulerSurgicalPreemption.java | 246 ++++++
.../capacity/TestContainerAllocation.java | 188 +++-
.../capacity/TestContainerResizing.java | 20 +-
.../TestNodeLabelContainerAllocation.java | 72 +-
...TestWorkPreservingRMRestartForNodeLabel.java | 6 +-
.../distributed/TestTopKNodeSelector.java | 147 ++++
.../fair/TestAllocationFileLoaderService.java | 22 +
.../scheduler/fair/TestFairScheduler.java | 4 +-
.../scheduler/fifo/TestFifoScheduler.java | 18 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
.../webapp/TestRMWebServicesNodeLabels.java | 107 +++
.../webapp/TestRMWebServicesNodes.java | 268 +++---
.../yarn/server/timeline/EntityCacheItem.java | 24 +-
.../timeline/EntityGroupFSTimelineStore.java | 22 +-
.../EntityGroupFSTimelineStoreMetrics.java | 160 ++++
.../timeline/LevelDBCacheTimelineStore.java | 2 +-
.../hadoop/yarn/server/timeline/LogInfo.java | 5 +-
.../TestEntityGroupFSTimelineStore.java | 22 +
.../src/site/markdown/ReservationSystem.md | 65 ++
.../src/site/markdown/ResourceManagerRest.md | 447 +++++++++-
.../hadoop-yarn-site/src/site/markdown/YARN.md | 2 +
.../images/yarn_reservation_system.png | Bin 0 -> 85449 bytes
pom.xml | 1 +
634 files changed, 30661 insertions(+), 16515 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a65c707c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a65c707c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a65c707c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index d390043,68e2537..4a446d4
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@@ -697,12 -699,18 +699,25 @@@ public class FsVolumeImpl implements Fs
} else {
ExtendedBlock block =
new ExtendedBlock(bpid, Block.filename2id(state.curEntry));
+ File expectedBlockDir = DatanodeUtil.idToBlockDir(
+ new File("."), block.getBlockId());
+ File actualBlockDir = Paths.get(".",
+ state.curFinalizedDir, state.curFinalizedSubDir).toFile();
+ if (!expectedBlockDir.equals(actualBlockDir)) {
+ LOG.error("nextBlock({}, {}): block id {} found in invalid " +
+ "directory. Expected directory: {}. " +
+ "Actual directory: {}", storageID, bpid,
+ block.getBlockId(), expectedBlockDir.getPath(),
+ actualBlockDir.getPath());
+ continue;
+ }
++
+ File blkFile = getBlockFile(bpid, block);
+ File metaFile = FsDatasetUtil.findMetaFile(blkFile);
+ block.setGenerationStamp(
+ Block.getGenerationStamp(metaFile.getName()));
+ block.setNumBytes(blkFile.length());
+
LOG.trace("nextBlock({}, {}): advancing to {}",
storageID, bpid, block);
return block;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org