You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/05/19 19:47:39 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76279f58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76279f58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76279f58
Branch: refs/heads/HDFS-7240
Commit: 76279f5877d592d2cc0816654f30cf1bfcf3aae1
Parents: 4770ae7 141873c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu May 19 12:44:16 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu May 19 12:44:16 2016 -0700
----------------------------------------------------------------------
BUILDING.txt | 8 +-
dev-support/docker/Dockerfile | 26 +-
hadoop-assemblies/pom.xml | 4 +-
.../main/resources/assemblies/hadoop-tools.xml | 56 ++
hadoop-build-tools/pom.xml | 2 +-
hadoop-client/pom.xml | 4 +-
.../hadoop-annotations/pom.xml | 4 +-
.../hadoop-auth-examples/pom.xml | 4 +-
hadoop-common-project/hadoop-auth/pom.xml | 4 +-
.../authentication/util/KerberosName.java | 9 +-
.../TestKerberosAuthenticationHandler.java | 7 +-
.../authentication/util/TestKerberosName.java | 17 +-
hadoop-common-project/hadoop-common/pom.xml | 4 +-
.../hadoop-common/src/main/bin/hadoop | 317 +++++---
.../src/main/bin/hadoop-functions.sh | 23 +
.../org/apache/hadoop/conf/Configuration.java | 21 +
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 89 ++-
.../apache/hadoop/crypto/key/KeyProvider.java | 32 +
.../org/apache/hadoop/crypto/key/KeyShell.java | 127 +--
.../fs/CommonConfigurationKeysPublic.java | 7 +-
.../hadoop/fs/EmptyStorageStatistics.java | 43 +
.../org/apache/hadoop/fs/FSDataInputStream.java | 9 +
.../java/org/apache/hadoop/fs/FileSystem.java | 58 +-
.../hadoop/fs/FileSystemStorageStatistics.java | 136 ++++
.../main/java/org/apache/hadoop/fs/FsShell.java | 15 +-
.../hadoop/fs/GlobalStorageStatistics.java | 130 +++
.../java/org/apache/hadoop/fs/QuotaUsage.java | 4 +-
.../org/apache/hadoop/fs/StorageStatistics.java | 93 +++
.../apache/hadoop/fs/TrashPolicyDefault.java | 1 -
.../hadoop/fs/UnionStorageStatistics.java | 113 +++
.../org/apache/hadoop/io/DataInputBuffer.java | 81 +-
.../main/java/org/apache/hadoop/io/IOUtils.java | 4 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 7 +-
.../hadoop/io/compress/DecompressorStream.java | 41 +-
.../hadoop/io/retry/FailoverProxyProvider.java | 11 +
.../hadoop/io/retry/RetryInvocationHandler.java | 350 +++++----
.../apache/hadoop/ipc/DecayRpcScheduler.java | 15 +-
.../java/org/apache/hadoop/log/LogLevel.java | 16 +-
.../hadoop/metrics2/MetricStringBuilder.java | 141 ++++
.../hadoop/metrics2/lib/MutableCounterLong.java | 2 +-
.../org/apache/hadoop/net/NetworkTopology.java | 144 +++-
.../java/org/apache/hadoop/net/NodeBase.java | 9 +
.../java/org/apache/hadoop/security/KDiag.java | 46 +-
.../hadoop/security/LdapGroupsMapping.java | 269 +++++--
.../apache/hadoop/security/ProviderUtils.java | 75 ++
.../hadoop/security/ShellBasedIdMapping.java | 21 +-
.../hadoop/security/UserGroupInformation.java | 6 +-
.../alias/AbstractJavaKeyStoreProvider.java | 113 +--
.../security/alias/CredentialProvider.java | 37 +-
.../hadoop/security/alias/CredentialShell.java | 166 ++--
.../security/alias/JavaKeyStoreProvider.java | 6 +-
.../alias/LocalJavaKeyStoreProvider.java | 4 +-
.../security/http/RestCsrfPreventionFilter.java | 12 +-
.../security/http/XFrameOptionsFilter.java | 167 ++++
.../hadoop/security/http/package-info.java | 22 +
.../org/apache/hadoop/security/token/Token.java | 2 +-
.../AbstractDelegationTokenIdentifier.java | 33 +-
.../DelegationTokenAuthenticationFilter.java | 7 +-
.../token/delegation/web/ServletUtils.java | 7 +-
.../org/apache/hadoop/util/SysInfoLinux.java | 39 +-
.../src/main/resources/core-default.xml | 68 +-
.../src/site/markdown/CommandsManual.md | 22 +-
.../src/site/markdown/FileSystemShell.md | 5 +
.../src/site/markdown/GroupsMapping.md | 6 +
.../src/site/markdown/SecureMode.md | 6 +
.../src/site/markdown/UnixShellGuide.md | 87 +-
.../crypto/key/TestKeyProviderFactory.java | 2 +-
.../apache/hadoop/crypto/key/TestKeyShell.java | 33 +-
.../apache/hadoop/fs/TestFilterFileSystem.java | 1 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 1 +
.../fs/contract/AbstractBondedFSContract.java | 3 +-
.../hadoop/io/TestSequenceFileAppend.java | 67 ++
.../hadoop/io/compress/FakeCompressor.java | 113 +++
.../hadoop/io/compress/FakeDecompressor.java | 109 +++
.../compress/TestBlockDecompressorStream.java | 180 +----
.../io/compress/TestDecompressorStream.java | 99 +++
.../apache/hadoop/io/retry/TestRetryProxy.java | 67 +-
.../org/apache/hadoop/log/TestLogLevel.java | 39 +-
.../org/apache/hadoop/security/TestKDiag.java | 16 +
.../hadoop/security/TestLdapGroupsMapping.java | 33 +-
.../security/TestLdapGroupsMappingBase.java | 71 +-
.../TestLdapGroupsMappingWithOneQuery.java | 100 +++
.../TestLdapGroupsMappingWithPosixGroup.java | 41 +-
.../security/TestUserGroupInformation.java | 27 +-
.../hadoop/security/alias/TestCredShell.java | 56 +-
.../alias/TestCredentialProviderFactory.java | 5 +-
.../http/TestRestCsrfPreventionFilter.java | 40 +-
.../security/http/TestXFrameOptionsFilter.java | 151 ++++
.../java/org/apache/hadoop/util/JarFinder.java | 26 +-
.../java/org/apache/hadoop/util/TestRunJar.java | 29 +-
.../apache/hadoop/util/TestSysInfoLinux.java | 92 ++-
.../scripts/hadoop-functions_test_helper.bash | 4 +-
.../src/test/scripts/hadoop_escape_chars.bats | 32 +
.../src/test/scripts/hadoop_subcommands.bats | 78 ++
hadoop-common-project/hadoop-kms/pom.xml | 4 +-
.../hadoop-kms/src/main/sbin/kms.sh | 14 +-
hadoop-common-project/hadoop-minikdc/pom.xml | 4 +-
hadoop-common-project/hadoop-nfs/pom.xml | 4 +-
hadoop-common-project/pom.xml | 4 +-
hadoop-dist/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +-
.../hadoop/hdfs/AsyncDistributedFileSystem.java | 59 ++
.../org/apache/hadoop/hdfs/ClientContext.java | 47 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 2 +-
.../hadoop/hdfs/DFSOpsCountStatistics.java | 167 ++++
.../org/apache/hadoop/hdfs/DataStreamer.java | 32 +-
.../hadoop/hdfs/DistributedFileSystem.java | 81 +-
.../apache/hadoop/hdfs/StripedDataStreamer.java | 2 +-
.../hadoop/hdfs/protocol/DatanodeInfo.java | 59 +-
.../ClientNamenodeProtocolTranslatorPB.java | 39 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 8 +
.../delegation/DelegationTokenIdentifier.java | 27 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 51 ++
.../src/main/proto/hdfs.proto | 2 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +-
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 8 +-
.../hadoop-hdfs-native-client/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +-
.../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 6 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 42 +-
.../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 4 +-
.../hadoop-hdfs/src/main/bin/hdfs | 445 ++++++-----
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +-
.../hdfs/client/CreateEncryptionZoneFlag.java | 71 ++
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 89 ++-
.../apache/hadoop/hdfs/client/package-info.java | 27 +
.../AvailableSpaceBlockPlacementPolicy.java | 11 +-
.../BlockPlacementPolicyDefault.java | 84 +-
.../CacheReplicationMonitor.java | 2 +-
.../hadoop/hdfs/server/common/Storage.java | 65 +-
.../hdfs/server/datanode/DataStorage.java | 2 +-
.../erasurecode/StripedReconstructor.java | 4 +-
.../datanode/metrics/DataNodeMetrics.java | 14 +
.../hadoop/hdfs/server/namenode/NNStorage.java | 25 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 4 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 +-
.../namenode/RedundantEditLogInputStream.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 13 +-
.../apache/hadoop/hdfs/tools/CacheAdmin.java | 6 +-
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 69 +-
.../hdfs/tools/DelegationTokenFetcher.java | 32 +-
.../hadoop/hdfs/tools/erasurecode/ECCli.java | 14 +-
.../StatisticsEditsVisitor.java | 11 +-
.../src/main/shellprofile.d/hadoop-hdfs.sh | 36 +
.../hadoop-hdfs/src/main/shellprofile.d/hdfs.sh | 36 -
.../site/markdown/CentralizedCacheManagement.md | 2 +-
.../src/site/markdown/HDFSCommands.md | 5 +-
.../markdown/HDFSHighAvailabilityWithQJM.md | 2 +-
.../src/site/markdown/TransparentEncryption.md | 13 +-
.../org/apache/hadoop/fs/TestGlobPaths.java | 19 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 278 +++----
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 31 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 32 +-
.../apache/hadoop/hdfs/TestAsyncDFSRename.java | 267 ++++++-
.../apache/hadoop/hdfs/TestDFSPermission.java | 29 +-
.../org/apache/hadoop/hdfs/TestDataStream.java | 85 ++
.../hadoop/hdfs/TestDataTransferProtocol.java | 4 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 227 +++++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 206 +++--
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 7 +-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 4 +-
.../org/apache/hadoop/hdfs/TestFileAppend.java | 4 +
.../apache/hadoop/hdfs/TestFileCorruption.java | 6 +-
.../java/org/apache/hadoop/hdfs/TestHFlush.java | 10 +-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 4 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 27 +-
.../org/apache/hadoop/hdfs/TestReplication.java | 3 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 12 +-
.../hdfs/security/TestDelegationToken.java | 2 +-
.../blockmanagement/TestBlockManager.java | 4 +-
.../blockmanagement/TestHeartbeatHandling.java | 14 +-
.../TestOverReplicatedBlocks.java | 8 +-
.../server/datanode/BlockReportTestBase.java | 2 +-
.../hdfs/server/datanode/DataNodeTestUtils.java | 119 +--
.../datanode/InternalDataNodeTestUtils.java | 157 ++++
.../TestDataNodeErasureCodingMetrics.java | 153 ++++
.../datanode/TestDataNodeHotSwapVolumes.java | 2 +-
.../datanode/TestDataNodeMetricsLogger.java | 2 +-
.../datanode/TestDataNodeReconfiguration.java | 2 +-
.../TestDataNodeVolumeFailureReporting.java | 36 +
.../hdfs/server/datanode/TestDataStorage.java | 26 +
...TestDnRespectsBlockReportSplitThreshold.java | 6 +-
.../server/datanode/TestFsDatasetCache.java | 2 +-
.../datanode/TestIncrementalBlockReports.java | 2 +-
.../hdfs/server/datanode/TestStorageReport.java | 2 +-
.../server/datanode/TestTriggerBlockReport.java | 2 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 2 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 12 +-
.../hdfs/server/namenode/TestDeleteRace.java | 4 +-
.../server/namenode/TestNameNodeMXBean.java | 24 +-
.../hdfs/server/namenode/TestSaveNamespace.java | 33 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 5 +-
.../namenode/ha/TestPipelinesFailover.java | 6 +-
.../hdfs/tools/TestDelegationTokenFetcher.java | 17 +
.../TestOfflineEditsViewer.java | 30 +
.../apache/hadoop/net/TestNetworkTopology.java | 84 +-
.../src/test/resources/testCryptoConf.xml | 86 ++
.../test/resources/testErasureCodingConf.xml | 16 +
.../scripts/hdfs-functions_test_helper.bash | 58 ++
.../src/test/scripts/hdfs_subcommands.bats | 78 ++
.../hadoop-hdfs/src/test/scripts/run-bats.sh | 43 +
hadoop-hdfs-project/pom.xml | 4 +-
hadoop-mapreduce-project/bin/mapred | 230 +++---
.../hadoop-mapreduce-client-app/pom.xml | 4 +-
.../v2/app/local/LocalContainerAllocator.java | 2 +
.../v2/app/rm/RMContainerAllocator.java | 62 +-
.../v2/app/rm/RMContainerRequestor.java | 6 +
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 3 +-
.../app/local/TestLocalContainerAllocator.java | 43 +
.../v2/app/rm/TestRMContainerAllocator.java | 250 ++++++
.../hadoop-mapreduce-client-common/pom.xml | 4 +-
.../v2/jobhistory/FileNameIndexUtils.java | 2 +-
.../v2/jobhistory/TestFileNameIndexUtils.java | 25 +
.../hadoop-mapreduce-client-core/pom.xml | 4 +-
.../hadoop/mapred/LocatedFileStatusFetcher.java | 4 +-
.../lib/input/CompressedSplitLineReader.java | 5 +
.../org/apache/hadoop/mapreduce/tools/CLI.java | 35 +-
.../src/site/markdown/MapredCommands.md | 3 +-
.../hadoop/mapred/TestLineRecordReader.java | 29 +
.../lib/input/TestLineRecordReader.java | 29 +
.../compressedMultibyteDelimiter.txt.bz2 | Bin 0 -> 1096 bytes
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +-
.../hadoop-mapreduce-client-hs/pom.xml | 8 +-
.../mapreduce/v2/hs/HistoryFileManager.java | 57 +-
.../mapreduce/v2/hs/webapp/HsJobBlock.java | 2 +-
.../v2/hs/webapp/dao/AMAttemptInfo.java | 2 +-
...HistoryFileManagerInitWithNonRunningDFS.java | 75 ++
...estUnnecessaryBlockingOnHistoryFileInfo.java | 323 ++++++++
.../v2/hs/webapp/TestHsWebServicesJobs.java | 12 +-
.../hadoop-mapreduce-client-jobclient/pom.xml | 4 +-
.../hadoop/mapreduce/TestMRJobClient.java | 28 +
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 9 -
.../hadoop-mapreduce-client-nativetask/pom.xml | 4 +-
.../hadoop-mapreduce-client-shuffle/pom.xml | 4 +-
.../hadoop/mapred/FadvisedFileRegion.java | 2 +-
.../hadoop-mapreduce-client/pom.xml | 4 +-
.../hadoop-mapreduce-examples/pom.xml | 4 +-
hadoop-mapreduce-project/pom.xml | 4 +-
.../shellprofile.d/hadoop-mapreduce.sh | 41 +
.../shellprofile.d/mapreduce.sh | 41 -
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-minicluster/pom.xml | 4 +-
hadoop-project-dist/pom.xml | 37 +-
hadoop-project/pom.xml | 16 +-
hadoop-tools/hadoop-ant/pom.xml | 4 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 4 +-
.../main/shellprofile.d/hadoop-archive-logs.sh | 39 +
hadoop-tools/hadoop-archives/pom.xml | 4 +-
.../src/main/shellprofile.d/hadoop-archives.sh | 58 ++
.../hadoop-aws/dev-support/findbugs-exclude.xml | 357 +--------
hadoop-tools/hadoop-aws/pom.xml | 120 ++-
.../apache/hadoop/fs/s3/FileSystemStore.java | 4 +-
.../org/apache/hadoop/fs/s3/S3Credentials.java | 2 +
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 4 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 4 +
.../org/apache/hadoop/fs/s3a/Constants.java | 20 +-
.../hadoop/fs/s3a/S3AFastOutputStream.java | 11 +-
.../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 12 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 484 +++++++-----
.../apache/hadoop/fs/s3a/S3AInputStream.java | 342 ++++++--
.../hadoop/fs/s3a/S3AInstrumentation.java | 457 +++++++++++
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 40 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 167 +++-
.../hadoop/fs/contract/s3/S3Contract.java | 10 +
.../fs/contract/s3/TestS3ContractRootDir.java | 15 +
.../fs/contract/s3/TestS3ContractSeek.java | 9 +
.../hadoop/fs/contract/s3a/S3AContract.java | 9 +-
.../fs/contract/s3a/TestS3AContractRootDir.java | 39 +-
.../fs/contract/s3n/NativeS3Contract.java | 9 +-
.../apache/hadoop/fs/s3/TestS3Credentials.java | 3 +
.../hadoop/fs/s3a/TestS3AConfiguration.java | 71 +-
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 195 ++++-
.../fs/s3a/scale/TestS3ADeleteManyFiles.java | 1 -
.../scale/TestS3AInputStreamPerformance.java | 285 +++++++
.../Jets3tNativeS3FileSystemContractTest.java | 30 -
.../TestJets3tNativeS3FileSystemContract.java | 30 +
.../src/test/resources/log4j.properties | 3 +
hadoop-tools/hadoop-azure/pom.xml | 2 +-
hadoop-tools/hadoop-datajoin/pom.xml | 4 +-
hadoop-tools/hadoop-distcp/pom.xml | 4 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 32 +-
.../org/apache/hadoop/tools/OptionsParser.java | 49 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 16 +-
.../src/main/shellprofile.d/hadoop-distcp.sh | 62 ++
.../apache/hadoop/tools/TestDistCpViewFs.java | 93 +--
.../apache/hadoop/tools/TestOptionsParser.java | 19 +-
.../hadoop/tools/mapred/TestCopyMapper.java | 85 ++
hadoop-tools/hadoop-extras/pom.xml | 4 +-
.../src/main/shellprofile.d/hadoop-extras.sh | 39 +
hadoop-tools/hadoop-gridmix/pom.xml | 4 +-
.../src/main/shellprofile.d/hadoop-gridmix.sh | 36 +
.../src/site/markdown/GridMix.md.vm | 18 +-
hadoop-tools/hadoop-kafka/pom.xml | 4 +-
hadoop-tools/hadoop-openstack/pom.xml | 26 +-
hadoop-tools/hadoop-pipes/pom.xml | 4 +-
hadoop-tools/hadoop-rumen/pom.xml | 4 +-
.../src/main/shellprofile.d/hadoop-rumen.sh | 58 ++
.../hadoop-rumen/src/site/markdown/Rumen.md.vm | 40 +-
hadoop-tools/hadoop-sls/pom.xml | 4 +-
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 18 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 9 +
hadoop-tools/hadoop-streaming/pom.xml | 4 +-
.../apache/hadoop/streaming/DumpTypedBytes.java | 3 +-
.../hadoop/streaming/HadoopStreaming.java | 3 +-
.../apache/hadoop/streaming/LoadTypedBytes.java | 3 +-
.../src/main/shellprofile.d/hadoop-streaming.sh | 55 ++
.../src/site/markdown/HadoopStreaming.md.vm | 30 +-
hadoop-tools/hadoop-tools-dist/pom.xml | 4 +-
hadoop-tools/pom.xml | 4 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 394 ++++++----
.../dev-support/findbugs-exclude.xml | 2 +-
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +-
.../hadoop/yarn/api/ApplicationConstants.java | 7 +
.../hadoop/yarn/conf/YarnConfiguration.java | 67 +-
.../yarn/conf/TestYarnConfigurationFields.java | 14 +-
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../hadoop-yarn-applications/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 10 -
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 787 ++++++++++++-------
.../yarn/client/api/impl/TestAMRMClient.java | 13 +-
.../yarn/client/api/impl/TestAMRMProxy.java | 6 +-
.../api/impl/TestDistributedScheduling.java | 307 ++++++++
.../hadoop/yarn/client/cli/TestLogsCLI.java | 495 +++++++++++-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 +-
.../logaggregation/AggregatedLogFormat.java | 20 +
.../yarn/logaggregation/LogCLIHelpers.java | 396 +++++++---
.../src/main/resources/yarn-default.xml | 13 +
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 +-
.../pom.xml | 4 +-
...pplicationHistoryManagerOnTimelineStore.java | 43 +-
.../webapp/AHSWebServices.java | 266 ++++++-
...pplicationHistoryManagerOnTimelineStore.java | 41 +-
.../webapp/TestAHSWebServices.java | 203 ++++-
.../hadoop-yarn-server-common/pom.xml | 4 +-
.../protocolrecords/NodeHeartbeatResponse.java | 8 +
.../RegisterNodeManagerResponse.java | 6 +-
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 88 ++-
.../pb/RegisterNodeManagerResponsePBImpl.java | 46 +-
.../api/records/ContainerQueuingLimit.java | 44 ++
.../api/records/QueuedContainersStatus.java | 2 +-
.../impl/pb/ContainerQueuingLimitPBImpl.java | 80 ++
.../api/records/impl/pb/NodeStatusPBImpl.java | 14 +-
.../impl/pb/QueuedContainersStatusPBImpl.java | 6 +-
.../metrics/ApplicationMetricsConstants.java | 3 +
.../yarn/server/webapp/dao/ContainerInfo.java | 6 +
.../yarn_server_common_service_protos.proto | 8 +
.../hadoop-yarn-server-nodemanager/pom.xml | 4 +-
.../hadoop/yarn/server/nodemanager/Context.java | 4 +-
.../yarn/server/nodemanager/NodeManager.java | 8 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 258 ++++--
.../containermanager/AuxServices.java | 43 +-
.../AuxiliaryServiceWithCustomClassLoader.java | 201 +++++
.../containermanager/ContainerManager.java | 42 +
.../containermanager/ContainerManagerImpl.java | 43 +-
.../application/ApplicationImpl.java | 8 +-
.../runtime/DockerLinuxContainerRuntime.java | 47 ++
.../logaggregation/AppLogAggregatorImpl.java | 4 +
.../logaggregation/LogAggregationService.java | 12 +-
.../loghandler/NonAggregatingLogHandler.java | 4 +
.../queuing/QueuingContainerManagerImpl.java | 96 ++-
.../recovery/NMLeveldbStateStoreService.java | 26 +-
.../recovery/NMNullStateStoreService.java | 4 +
.../recovery/NMStateStoreService.java | 9 +
.../nodemanager/scheduler/LocalScheduler.java | 6 +-
.../OpportunisticContainerAllocator.java | 17 +-
.../nodemanager/webapp/NMWebServices.java | 22 +-
.../nodemanager/TestNodeStatusUpdater.java | 121 +++
.../amrmproxy/BaseAMRMProxyTest.java | 3 +-
.../containermanager/TestAuxServices.java | 119 ++-
.../TestContainerManagerRecovery.java | 84 ++
.../runtime/TestDockerContainerRuntime.java | 115 +++
.../TestLogAggregationService.java | 11 +-
.../TestNonAggregatingLogHandler.java | 58 +-
.../monitor/TestContainerMetrics.java | 11 -
.../recovery/NMMemoryStateStoreService.java | 6 +
.../TestNMLeveldbStateStoreService.java | 12 +
.../scheduler/TestLocalScheduler.java | 9 +-
.../nodemanager/webapp/TestNMWebServices.java | 12 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 +-
.../server/resourcemanager/ClientRMService.java | 11 +-
.../server/resourcemanager/ClusterMonitor.java | 6 +-
.../DistributedSchedulingService.java | 194 +++--
.../resourcemanager/NodesListManager.java | 120 ++-
.../yarn/server/resourcemanager/RMContext.java | 4 +
.../server/resourcemanager/RMContextImpl.java | 14 +
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../server/resourcemanager/ResourceManager.java | 2 +
.../resourcemanager/ResourceTrackerService.java | 70 +-
.../resourcemanager/amlauncher/AMLauncher.java | 14 +
.../metrics/ApplicaitonStateUpdatedEvent.java | 47 ++
.../metrics/ContainerFinishedEvent.java | 9 +-
.../metrics/SystemMetricsEventType.java | 1 +
.../metrics/SystemMetricsPublisher.java | 97 ++-
.../monitor/SchedulingMonitor.java | 16 +-
.../CapacitySchedulerPreemptionUtils.java | 35 +-
.../capacity/FifoCandidatesSelector.java | 26 +-
.../capacity/PreemptableResourceCalculator.java | 139 +++-
.../ProportionalCapacityPreemptionPolicy.java | 96 ++-
.../ReservedContainerCandidatesSelector.java | 316 ++++++++
.../monitor/capacity/TempQueuePerPartition.java | 174 +++-
.../server/resourcemanager/rmapp/RMAppImpl.java | 19 +-
.../rmcontainer/RMContainer.java | 2 +
.../rmcontainer/RMContainerImpl.java | 10 +
.../server/resourcemanager/rmnode/RMNode.java | 5 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 53 +-
.../scheduler/AbstractYarnScheduler.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 4 +
.../scheduler/SchedulerNode.java | 29 +-
.../scheduler/capacity/CapacityScheduler.java | 10 +-
.../CapacitySchedulerConfiguration.java | 24 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 1 +
.../common/fica/FiCaSchedulerNode.java | 7 +-
.../distributed/NodeQueueLoadMonitor.java | 273 +++++++
.../distributed/QueueLimitCalculator.java | 125 +++
.../scheduler/distributed/TopKNodeSelector.java | 223 ------
.../scheduler/fair/FSAppAttempt.java | 1 +
.../scheduler/fair/FairScheduler.java | 3 +-
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../security/DelegationTokenRenewer.java | 9 +-
.../security/QueueACLsManager.java | 32 +-
.../webapp/FairSchedulerPage.java | 1 +
.../resourcemanager/webapp/RMWebServices.java | 30 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 9 +
.../yarn/server/resourcemanager/MockNM.java | 21 +-
.../yarn/server/resourcemanager/MockNodes.java | 9 +
.../resourcemanager/TestApplicationACLs.java | 5 +-
.../TestApplicationMasterLauncher.java | 5 +
.../resourcemanager/TestClientRMService.java | 15 +-
.../TestDistributedSchedulingService.java | 68 +-
.../resourcemanager/TestRMAdminService.java | 99 +++
.../TestResourceTrackerService.java | 483 +++++++++++-
.../TestWorkPreservingRMRestart.java | 72 +-
.../metrics/TestSystemMetricsPublisher.java | 59 +-
...alCapacityPreemptionPolicyMockFramework.java | 689 ++++++++++++++++
...estProportionalCapacityPreemptionPolicy.java | 62 +-
...pacityPreemptionPolicyForNodePartitions.java | 687 +---------------
...tyPreemptionPolicyForReservedContainers.java | 430 ++++++++++
...alCapacityPreemptionPolicyMockFramework.java | 247 ++++++
.../TestSchedulerApplicationAttempt.java | 3 +-
.../CapacitySchedulerPreemptionTestBase.java | 149 ++++
.../capacity/TestCapacityScheduler.java | 2 +-
.../TestCapacitySchedulerLazyPreemption.java | 638 +++++++++++++++
.../TestCapacitySchedulerPreemption.java | 677 ----------------
...TestCapacitySchedulerSurgicalPreemption.java | 246 ++++++
.../distributed/TestNodeQueueLoadMonitor.java | 195 +++++
.../distributed/TestTopKNodeSelector.java | 147 ----
.../scheduler/fair/TestFairScheduler.java | 4 +-
.../scheduler/fifo/TestFifoScheduler.java | 7 +-
.../webapp/TestRMWebServicesNodeLabels.java | 107 +++
.../webapp/TestRMWebServicesNodes.java | 12 +-
.../pom.xml | 4 +-
.../hadoop-yarn-server-tests/pom.xml | 4 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 67 +-
.../pom.xml | 4 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 4 +-
.../server/webproxy/WebAppProxyServlet.java | 11 +-
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +-
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 +-
.../hadoop-yarn/shellprofile.d/hadoop-yarn.sh | 62 ++
.../hadoop-yarn/shellprofile.d/yarn.sh | 62 --
hadoop-yarn-project/pom.xml | 4 +-
pom.xml | 5 +-
466 files changed, 20139 insertions(+), 6382 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76279f58/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76279f58/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76279f58/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index d0e665d,2a29d17..da4be41
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@@ -58,7 -61,194 +63,200 @@@ function hadoop_usag
hadoop_add_subcommand "storagepolicies" "list/get/set block storage policies"
hadoop_add_subcommand "version" "print the version"
hadoop_add_subcommand "zkfc" "run the ZK Failover Controller daemon"
- hadoop_generate_usage "${MYNAME}" false
+ hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
+ }
+
+ ## @description Default command handler for hadoop command
+ ## @audience public
+ ## @stability stable
+ ## @replaceable no
+ ## @param CLI arguments
+ function hdfscmd_case
+ {
+ subcmd=$1
+ shift
+
+ case ${subcmd} in
+ balancer)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.balancer.Balancer
+ hadoop_debug "Appending HADOOP_BALANCER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}"
+ ;;
+ cacheadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin
+ ;;
+ classpath)
+ hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
+ ;;
+ crypto)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CryptoAdmin
+ ;;
+ datanode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ # Determine if we're starting a secure datanode, and
+ # if so, redefine appropriate variables
+ if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
+ HADOOP_SUBCMD_SECURESERVICE="true"
+ HADOOP_SUBCMD_SECUREUSER="${HADOOP_SECURE_DN_USER}"
+
+ # backward compatiblity
+ HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}"
+ HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}"
+
+ hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
+ hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}"
+ HADOOP_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
+ else
+ hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
+ fi
+ ;;
+ debug)
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DebugAdmin'
+ ;;
+ dfs)
+ HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ dfsadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSAdmin
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ envvars)
+ echo "JAVA_HOME='${JAVA_HOME}'"
+ echo "HADOOP_HDFS_HOME='${HADOOP_HDFS_HOME}'"
+ echo "HDFS_DIR='${HDFS_DIR}'"
+ echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'"
+ echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
+ echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+ echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+ echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
+ exit 0
+ ;;
+ erasurecode)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ fetchdt)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
+ ;;
+ fsck)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ getconf)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetConf
+ ;;
+ groups)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetGroups
+ ;;
+ haadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSHAAdmin
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ journalnode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
+ hadoop_debug "Appending HADOOP_JOURNALNODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}"
+ ;;
+ jmxget)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
+ ;;
+ lsSnapshottableDir)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
+ ;;
+ mover)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.mover.Mover
+ hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}"
+ ;;
+ namenode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.NameNode'
+ hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}"
+ hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
+ ;;
+ nfs3)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then
+ HADOOP_SUBCMD_SECURESERVICE="true"
+ HADOOP_SUBCMD_SECUREUSER="${HADOOP_PRIVILEGED_NFS_USER}"
+
+ # backward compatiblity
+ HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}"
+ HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}"
+
+ hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
+ hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter
+ else
+ hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3
+ fi
+ ;;
+ oev)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
+ ;;
+ oiv)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
+ ;;
+ oiv_legacy)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
+ ;;
+ portmap)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap
+ hadoop_debug "Appending HADOOP_PORTMAP_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}"
+ ;;
++ scm)
++ supportdaemonization="true"
++ CLASS='org.apache.hadoop.ozone.storage.StorageContainerManager'
++ hadoop_debug "Appending HADOOP_SCM_OPTS onto HADOOP_OPTS"
++ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SCM_OPTS}"
++ ;;
+ secondarynamenode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
+ hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}"
+ hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
+ ;;
+ snapshotDiff)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
+ ;;
+ storagepolicies)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
+ ;;
+ version)
+ HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
+ ;;
+ zkfc)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
+ hadoop_debug "Appending HADOOP_ZKFC_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}"
+ ;;
+ *)
+ HADOOP_CLASSNAME="${subcmd}"
+ if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
+ hadoop_exit_with_usage 1
+ fi
+ ;;
+ esac
}
# let's locate libexec...
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76279f58/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org