You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/08/31 16:41:29 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into
HADOOP-13345
Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11e5f54f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11e5f54f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11e5f54f
Branch: refs/heads/HADOOP-13345
Commit: 11e5f54fee519980dcd82211125b15cb9bfeb0e4
Parents: 845cd36 f9e0cc8
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 31 16:56:02 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 31 16:56:02 2017 +0100
----------------------------------------------------------------------
BUILDING.txt | 4 +-
dev-support/docker/Dockerfile | 17 +-
.../authentication/client/AuthenticatedURL.java | 184 +-
.../client/KerberosAuthenticator.java | 30 +-
.../client/PseudoAuthenticator.java | 5 +-
.../hadoop-common/HadoopCommon.cmake | 26 +-
.../hadoop-common/src/CMakeLists.txt | 2 +-
.../src/main/conf/hadoop-policy.xml | 11 +
.../org/apache/hadoop/conf/Configuration.java | 71 +-
.../crypto/key/KeyProviderCryptoExtension.java | 147 +-
.../crypto/key/kms/KMSClientProvider.java | 158 +-
.../hadoop/crypto/key/kms/KMSRESTConstants.java | 1 +
.../key/kms/LoadBalancingKMSClientProvider.java | 20 +
.../hadoop/fs/FSDataOutputStreamBuilder.java | 74 +-
.../org/apache/hadoop/fs/FsServerDefaults.java | 9 +
.../org/apache/hadoop/fs/LocatedFileStatus.java | 6 +-
.../hadoop/fs/permission/FsPermission.java | 2 +-
.../apache/hadoop/fs/sftp/SFTPFileSystem.java | 6 +
.../org/apache/hadoop/fs/shell/AclCommands.java | 6 +-
.../hadoop/fs/shell/CommandWithDestination.java | 4 +-
.../java/org/apache/hadoop/fs/shell/Ls.java | 4 +-
.../org/apache/hadoop/http/HttpServer2.java | 9 +-
.../hadoop/io/erasurecode/CodecRegistry.java | 2 +-
.../io/erasurecode/ErasureCodeConstants.java | 8 +
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 14 +-
.../java/org/apache/hadoop/util/KMSUtil.java | 134 ++
.../hadoop/util/curator/ZKCuratorManager.java | 142 +-
.../bzip2/org_apache_hadoop_io_compress_bzip2.h | 2 +
.../src/main/resources/core-default.xml | 10 +-
.../hadoop-common/src/site/markdown/Metrics.md | 24 +-
.../src/site/markdown/filesystem/filesystem.md | 33 +-
.../filesystem/fsdataoutputstreambuilder.md | 182 ++
.../src/site/markdown/filesystem/index.md | 1 +
.../conf/TestCommonConfigurationFields.java | 1 +
.../apache/hadoop/conf/TestConfiguration.java | 390 ++--
.../hadoop/conf/TestConfigurationSubclass.java | 8 +-
.../apache/hadoop/conf/TestDeprecatedKeys.java | 8 +-
.../apache/hadoop/conf/TestGetInstances.java | 8 +-
.../key/TestKeyProviderCryptoExtension.java | 113 +-
.../org/apache/hadoop/fs/TestAvroFSInput.java | 6 +-
.../test/java/org/apache/hadoop/fs/TestDU.java | 19 +-
.../java/org/apache/hadoop/fs/TestFilterFs.java | 8 +-
.../hadoop/fs/TestGetFileBlockLocations.java | 25 +-
.../org/apache/hadoop/fs/TestGlobExpander.java | 7 +-
.../apache/hadoop/fs/TestLocatedFileStatus.java | 52 +
.../java/org/apache/hadoop/fs/TestTrash.java | 24 +-
.../apache/hadoop/fs/TestTruncatedInputBug.java | 6 +-
.../hadoop/fs/permission/TestFsPermission.java | 14 +-
.../org/apache/hadoop/http/TestHttpServer.java | 13 +
.../hadoop/http/TestHttpServerWithSpengo.java | 242 ++-
.../apache/hadoop/ipc/TestFairCallQueue.java | 33 +-
.../org/apache/hadoop/log/TestLog4Json.java | 6 +-
.../hadoop/net/TestScriptBasedMapping.java | 9 +-
.../TestScriptBasedMappingWithDependency.java | 7 +-
.../security/TestAuthenticationFilter.java | 6 +-
.../TestAuthenticationWithProxyUserFilter.java | 7 +-
.../security/TestWhitelistBasedResolver.java | 9 +-
.../apache/hadoop/security/token/TestToken.java | 12 +-
.../hadoop/util/TestAsyncDiskService.java | 6 +-
.../apache/hadoop/util/TestCacheableIPList.java | 13 +-
.../apache/hadoop/util/TestFileBasedIPList.java | 12 +-
.../org/apache/hadoop/util/TestFindClass.java | 2 +-
.../apache/hadoop/util/TestGenericsUtil.java | 12 +-
.../org/apache/hadoop/util/TestIndexedSort.java | 7 +-
.../hadoop/util/TestNativeLibraryChecker.java | 8 +-
.../util/curator/TestZKCuratorManager.java | 41 +-
...rKeyGeneratorKeyProviderCryptoExtension.java | 6 +
.../hadoop/crypto/key/kms/server/KMS.java | 113 +-
.../crypto/key/kms/server/KMSJSONReader.java | 8 +-
.../key/kms/server/KMSServerJSONUtils.java | 34 +-
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 18 +
.../kms/server/KeyAuthorizationKeyProvider.java | 19 +
.../hadoop-kms/src/site/markdown/index.md.vm | 60 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 46 +-
.../crypto/key/kms/server/TestKMSAudit.java | 7 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 25 +-
.../hadoop/hdfs/DFSClientFaultInjector.java | 2 +
.../org/apache/hadoop/hdfs/DFSInputStream.java | 153 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 58 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 3 +-
.../hadoop/hdfs/DistributedFileSystem.java | 88 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 46 +-
.../hdfs/client/HdfsClientConfigKeys.java | 2 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 35 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 55 +-
.../hdfs/protocol/ErasureCodingPolicyState.java | 73 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 7 +
.../hdfs/protocol/ReencryptionStatus.java | 216 ++
.../protocol/ReencryptionStatusIterator.java | 58 +
.../protocol/SystemErasureCodingPolicies.java | 18 +-
.../hdfs/protocol/ZoneReencryptionStatus.java | 257 +++
.../ClientNamenodeProtocolTranslatorPB.java | 44 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 154 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 4 +
.../src/main/proto/encryption.proto | 41 +
.../src/main/proto/hdfs.proto | 24 +
.../fs/http/client/BaseTestHttpFSWith.java | 1 +
.../src/CMakeLists.txt | 2 +-
.../src/main/native/fuse-dfs/CMakeLists.txt | 2 +
.../main/native/libhdfs-tests/CMakeLists.txt | 2 +
.../src/main/native/libhdfs/CMakeLists.txt | 3 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 17 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 39 +-
.../hdfs/qjournal/server/JournalNodeSyncer.java | 23 +-
.../blockmanagement/DatanodeDescriptor.java | 6 +-
.../server/blockmanagement/DatanodeManager.java | 45 +-
.../hdfs/server/datanode/BlockReceiver.java | 3 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 7 +-
.../hdfs/server/datanode/DirectoryScanner.java | 5 +-
.../namenode/CheckpointFaultInjector.java | 18 +-
.../namenode/EncryptionFaultInjector.java | 9 +
.../server/namenode/EncryptionZoneManager.java | 351 +++-
.../namenode/ErasureCodingPolicyManager.java | 202 +-
.../hdfs/server/namenode/FSDirAttrOp.java | 4 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 238 ++-
.../server/namenode/FSDirErasureCodingOp.java | 44 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 6 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 7 +
.../hdfs/server/namenode/FSDirectory.java | 11 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 4 +-
.../hdfs/server/namenode/FSNamesystem.java | 95 +-
.../hadoop/hdfs/server/namenode/INode.java | 18 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 31 +-
.../server/namenode/ReencryptionHandler.java | 940 +++++++++
.../server/namenode/ReencryptionUpdater.java | 523 +++++
.../server/namenode/ha/StandbyCheckpointer.java | 16 +-
.../namenode/snapshot/SnapshotManager.java | 28 +
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 134 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 36 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 40 +-
.../src/main/resources/hdfs-default.xml | 98 +-
.../src/main/webapps/hdfs/explorer.js | 6 +
.../src/site/markdown/HDFSErasureCoding.md | 24 +-
.../src/site/markdown/TransparentEncryption.md | 45 +-
.../apache/hadoop/cli/TestErasureCodingCLI.java | 2 +-
.../TestClientProtocolForPipelineRecovery.java | 47 +
.../apache/hadoop/hdfs/TestDFSInputStream.java | 41 +
.../org/apache/hadoop/hdfs/TestDFSShell.java | 24 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 2 -
.../apache/hadoop/hdfs/TestEncryptionZones.java | 47 +
.../hadoop/hdfs/TestErasureCodingPolicies.java | 114 +-
.../java/org/apache/hadoop/hdfs/TestPread.java | 26 +-
.../apache/hadoop/hdfs/TestRollingUpgrade.java | 48 +
.../org/apache/hadoop/hdfs/TestSafeMode.java | 25 +
.../qjournal/server/TestJournalNodeSync.java | 176 +-
.../blockmanagement/TestBlockManager.java | 89 +-
.../blockmanagement/TestDatanodeManager.java | 96 +-
.../server/datanode/TestDirectoryScanner.java | 60 +-
.../hdfs/server/namenode/FSAclBaseTest.java | 6 +
.../namenode/TestEncryptionZoneManager.java | 26 +
.../hdfs/server/namenode/TestFSDirAttrOp.java | 7 +
.../hdfs/server/namenode/TestFSImage.java | 87 +
.../hdfs/server/namenode/TestReencryption.java | 1847 ++++++++++++++++++
.../namenode/TestReencryptionHandler.java | 197 ++
.../server/namenode/TestStripedINodeFile.java | 27 +-
.../snapshot/TestOpenFilesWithSnapshot.java | 221 +++
.../snapshot/TestRandomOpsWithSnapshots.java | 691 +++++++
.../snapshot/TestSnapshotDiffReport.java | 163 +-
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 182 +-
.../src/test/resources/testCryptoConf.xml | 112 ++
.../test/resources/testErasureCodingConf.xml | 179 +-
.../src/test/resources/test_ec_policies.xml | 2 +-
.../jobhistory/JobHistoryEventHandler.java | 105 +-
.../hadoop/mapreduce/jobhistory/JobSummary.java | 49 +-
.../apache/hadoop/mapreduce/v2/app/job/Job.java | 4 +
.../mapreduce/v2/app/job/impl/JobImpl.java | 43 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 58 +-
.../mapreduce/v2/app/job/impl/TaskImpl.java | 19 +-
.../v2/app/rm/RMContainerAllocator.java | 14 +-
.../hadoop/mapreduce/jobhistory/TestEvents.java | 4 +-
.../jobhistory/TestJobHistoryEventHandler.java | 49 +-
.../mapreduce/jobhistory/TestJobSummary.java | 6 +-
.../hadoop/mapreduce/v2/app/MockJobs.java | 19 +
.../mapreduce/v2/app/TestRuntimeEstimators.java | 20 +
.../v2/app/rm/TestRMContainerAllocator.java | 136 ++
.../hadoop-mapreduce-client-common/pom.xml | 4 -
.../mapred/TestMRWithDistributedCache.java | 40 +-
.../src/main/avro/Events.avpr | 10 +-
.../ClientDistributedCacheManager.java | 6 +-
.../mapreduce/jobhistory/HistoryViewer.java | 18 +-
.../HumanReadableHistoryViewerPrinter.java | 4 +-
.../jobhistory/JSONHistoryViewerPrinter.java | 4 +-
.../mapreduce/jobhistory/JobFinishedEvent.java | 55 +-
.../mapreduce/jobhistory/JobHistoryParser.java | 34 +-
.../JobUnsuccessfulCompletionEvent.java | 78 +-
.../jobhistory/MapAttemptFinishedEvent.java | 87 +-
.../jobhistory/ReduceAttemptFinishedEvent.java | 83 +-
.../jobhistory/TaskAttemptFinishedEvent.java | 47 +-
.../TaskAttemptUnsuccessfulCompletionEvent.java | 76 +-
.../mapreduce/jobhistory/TaskFailedEvent.java | 51 +-
.../mapreduce/jobhistory/TaskFinishedEvent.java | 42 +-
.../hadoop/mapred/TestFileOutputCommitter.java | 23 +-
.../apache/hadoop/mapred/TestIndexCache.java | 15 +-
.../hadoop/mapred/TestJobEndNotifier.java | 12 +-
.../hadoop/mapred/TestMapFileOutputFormat.java | 6 +-
.../mapreduce/TestJobMonitorAndPrint.java | 8 +-
.../jobhistory/TestHistoryViewerPrinter.java | 4 +-
.../lib/output/TestFileOutputCommitter.java | 39 +-
.../lib/output/TestFileOutputFormat.java | 7 +-
.../lib/output/TestMapFileOutputFormat.java | 2 +
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 55 +-
.../hadoop/mapreduce/v2/hs/PartialJob.java | 19 +
.../hadoop/mapreduce/v2/hs/UnparsedJob.java | 20 +
.../mapreduce/v2/hs/TestJobHistoryParsing.java | 122 +-
.../v2/hs/webapp/TestHsWebServicesAcls.java | 20 +
.../java/org/apache/hadoop/fs/TestDFSIO.java | 11 +-
.../mapred/TestMRTimelineEventHandling.java | 30 +-
.../mapred/jobcontrol/TestJobControl.java | 6 +-
.../mapreduce/JobHistoryFileReplayMapperV1.java | 3 +-
.../hadoop/mapreduce/SimpleEntityWriterV1.java | 3 +-
.../hadoop/mapreduce/TestMapCollection.java | 6 -
.../lib/input/TestDelegatingInputFormat.java | 7 +-
.../lib/jobcontrol/TestMapReduceJobControl.java | 2 +-
.../output/TestMRCJCFileOutputCommitter.java | 17 +-
.../src/CMakeLists.txt | 2 +-
.../mapred/nativetask/TestTaskContext.java | 23 +-
.../nativetask/buffer/TestInputBuffer.java | 7 +-
.../nativetask/buffer/TestOutputBuffer.java | 7 +-
.../nativetask/serde/TestKVSerializer.java | 13 +-
.../nativetask/utils/TestReadWriteBuffer.java | 7 +-
.../nativetask/utils/TestSizedWritable.java | 5 +-
.../examples/TestBaileyBorweinPlouffe.java | 14 +-
.../hadoop/examples/pi/math/TestLongLong.java | 33 +-
.../hadoop/examples/pi/math/TestModular.java | 105 +-
.../hadoop/examples/pi/math/TestSummation.java | 30 +-
.../maven/plugin/cmakebuilder/CompileMojo.java | 2 -
hadoop-project/pom.xml | 2 +-
hadoop-project/src/site/markdown/index.md.vm | 8 +-
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 27 +
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 4 +
.../apache/hadoop/fs/adl/TestGetFileStatus.java | 1 +
.../fs/azure/AzureNativeFileSystemStore.java | 57 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 47 +-
.../fs/azure/SecureStorageInterfaceImpl.java | 29 +-
.../hadoop-azure/src/site/markdown/index.md | 9 +
.../TestFileSystemOperationsWithThreads.java | 61 +-
...stNativeAzureFSAuthWithBlobSpecificKeys.java | 44 +
...estNativeAzureFileSystemConcurrencyLive.java | 165 +-
.../hadoop/contrib/utils/join/TestDataJoin.java | 41 +-
.../hadoop/tools/CopyListingFileStatus.java | 4 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 4 +-
.../hadoop/tools/util/TestDistCpUtils.java | 26 +-
.../org/apache/hadoop/tools/TestDistCh.java | 23 +-
hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 2 +-
.../rumen/Job20LineHistoryEventEmitter.java | 6 +-
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 4 +
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 2 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 2 +-
.../hadoop-sls/src/test/resources/yarn-site.xml | 10 -
.../typedbytes/TestTypedBytesWritable.java | 7 +-
.../api/protocolrecords/AllocateResponse.java | 32 +-
.../hadoop/yarn/api/records/CollectorInfo.java | 59 +
.../timelineservice/ApplicationEntity.java | 28 +
.../records/timelineservice/TimelineEntity.java | 52 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 108 +-
.../hadoop/yarn/util/TimelineServiceHelper.java | 8 +
.../src/main/proto/yarn_protos.proto | 5 +
.../src/main/proto/yarn_service_protos.proto | 2 +-
.../timelineservice/TestApplicationEntity.java | 71 +
.../yarn/conf/TestYarnConfigurationFields.java | 10 +
.../distributedshell/ApplicationMaster.java | 42 +-
.../distributedshell/TestDistributedShell.java | 173 +-
.../hadoop/yarn/client/api/AMRMClient.java | 5 +-
.../api/async/impl/AMRMClientAsyncImpl.java | 19 +-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 3 +-
.../ApplicationMasterServiceProtoTestBase.java | 72 +
.../hadoop/yarn/client/ProtocolHATestBase.java | 20 +-
...ationMasterServiceProtocolForTimelineV2.java | 71 +
...estApplicationMasterServiceProtocolOnHA.java | 46 +-
.../api/async/impl/TestAMRMClientAsync.java | 2 +-
.../client/api/impl/BaseAMRMProxyE2ETest.java | 2 +-
.../impl/TestAMRMClientContainerRequest.java | 6 +-
.../yarn/client/api/impl/TestAMRMProxy.java | 10 +-
.../hadoop/yarn/client/cli/TestLogsCLI.java | 65 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 14 +-
.../impl/pb/AllocateResponsePBImpl.java | 37 +-
.../records/impl/pb/CollectorInfoPBImpl.java | 152 ++
.../yarn/client/api/TimelineV2Client.java | 10 +-
.../client/api/impl/TimelineV2ClientImpl.java | 117 +-
.../logaggregation/AggregatedLogFormat.java | 22 +-
.../logaggregation/LogAggregationUtils.java | 41 +
.../yarn/logaggregation/LogToolUtils.java | 52 +
.../LogAggregationFileController.java | 404 ++++
.../LogAggregationFileControllerContext.java | 130 ++
.../LogAggregationFileControllerFactory.java | 195 ++
.../LogAggregationTFileController.java | 127 ++
.../filecontroller/package-info.java | 21 +
.../hadoop/yarn/util/BoundedAppender.java | 142 ++
.../src/main/resources/yarn-default.xml | 71 +
.../hadoop/yarn/api/TestPBImplRecords.java | 2 +
.../api/impl/TestTimelineClientV2Impl.java | 56 +-
.../logaggregation/TestAggregatedLogsBlock.java | 28 +-
.../logaggregation/TestContainerLogsUtils.java | 29 +-
...TestLogAggregationFileControllerFactory.java | 171 ++
.../hadoop/yarn/util/TestBoundedAppender.java | 115 ++
.../ApplicationHistoryServer.java | 79 +-
.../security/TimelineAuthenticationFilter.java | 49 -
...TimelineAuthenticationFilterInitializer.java | 129 --
...lineDelegationTokenSecretManagerService.java | 240 ---
...neV1DelegationTokenSecretManagerService.java | 225 +++
.../TestTimelineAuthenticationFilter.java | 323 ---
.../TestTimelineAuthenticationFilterForV1.java | 332 ++++
...TimelineAuthenticationFilterInitializer.java | 76 -
.../hadoop-yarn-server-common/pom.xml | 5 +
.../protocolrecords/NodeHeartbeatRequest.java | 13 +-
.../protocolrecords/NodeHeartbeatResponse.java | 14 +-
.../ReportNewCollectorInfoRequest.java | 13 +-
.../impl/pb/NodeHeartbeatRequestPBImpl.java | 74 +-
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 103 +-
.../pb/ReportNewCollectorInfoRequestPBImpl.java | 36 +-
.../server/api/records/AppCollectorData.java | 125 ++
.../server/api/records/AppCollectorsMap.java | 46 -
.../records/impl/pb/AppCollectorDataPBImpl.java | 227 +++
.../records/impl/pb/AppCollectorsMapPBImpl.java | 152 --
.../api/records/impl/pb/package-info.java | 19 +
.../store/impl/SQLFederationStateStore.java | 79 +
.../impl/ZookeeperFederationStateStore.java | 634 ++++++
.../FederationStateStoreClientMetrics.java | 184 ++
.../federation/store/metrics/package-info.java | 17 +
.../security/TimelineAuthenticationFilter.java | 55 +
...TimelineAuthenticationFilterInitializer.java | 137 ++
...elineDelgationTokenSecretManagerService.java | 83 +
.../server/timeline/security/package-info.java | 26 +
.../server/uam/UnmanagedApplicationManager.java | 2 +-
.../util/timeline/TimelineServerUtils.java | 92 +
.../yarn/server/util/timeline/package-info.java | 25 +
.../yarn_server_common_service_protos.proto | 19 +-
.../java/org/apache/hadoop/yarn/TestRPC.java | 36 +-
.../hadoop/yarn/TestYarnServerApiClasses.java | 54 +-
.../impl/TestZookeeperFederationStateStore.java | 89 +
.../TestFederationStateStoreClientMetrics.java | 146 ++
.../TestFederationStateStoreFacadeRetry.java | 20 +-
...TimelineAuthenticationFilterInitializer.java | 78 +
.../src/CMakeLists.txt | 34 +-
.../CMgrDecreaseContainersResourceEvent.java | 37 -
.../nodemanager/CMgrUpdateContainersEvent.java | 48 +
.../server/nodemanager/ContainerExecutor.java | 10 +-
.../nodemanager/ContainerManagerEventType.java | 2 +-
.../hadoop/yarn/server/nodemanager/Context.java | 14 +-
.../nodemanager/DefaultContainerExecutor.java | 12 +-
.../server/nodemanager/DeletionService.java | 7 +-
.../server/nodemanager/DirectoryCollection.java | 68 +-
.../nodemanager/LinuxContainerExecutor.java | 8 +-
.../nodemanager/LocalDirsHandlerService.java | 66 +-
.../yarn/server/nodemanager/NMAuditLogger.java | 7 +-
.../yarn/server/nodemanager/NodeManager.java | 77 +-
.../nodemanager/NodeResourceMonitorImpl.java | 8 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 74 +-
.../amrmproxy/AMRMProxyTokenSecretManager.java | 8 +-
.../nodemanager/api/impl/pb/NMProtoUtils.java | 7 +-
.../collectormanager/NMCollectorService.java | 50 +-
.../containermanager/AuxServices.java | 14 +-
.../containermanager/ContainerManagerImpl.java | 68 +-
.../ApplicationContainerFinishedEvent.java | 9 +-
.../application/ApplicationImpl.java | 40 +-
.../containermanager/container/Container.java | 2 +
.../container/ContainerImpl.java | 31 +-
.../deletion/task/DeletionTask.java | 7 +-
.../launcher/ContainerLaunch.java | 50 +-
.../launcher/ContainerRelaunch.java | 7 +-
.../launcher/ContainersLauncher.java | 7 +-
.../launcher/RecoveredContainerLaunch.java | 8 +-
.../privileged/PrivilegedOperationExecutor.java | 7 +-
.../CGroupsBlkioResourceHandlerImpl.java | 8 +-
.../CGroupsCpuResourceHandlerImpl.java | 7 +-
.../linux/resources/CGroupsHandlerImpl.java | 9 +-
.../CGroupsMemoryResourceHandlerImpl.java | 8 +-
.../linux/resources/ResourceHandlerModule.java | 13 +-
.../TrafficControlBandwidthHandlerImpl.java | 10 +-
.../linux/resources/TrafficController.java | 7 +-
.../runtime/DefaultLinuxContainerRuntime.java | 8 +-
.../DelegatingLinuxContainerRuntime.java | 8 +-
.../runtime/DockerLinuxContainerRuntime.java | 8 +-
.../JavaSandboxLinuxContainerRuntime.java | 13 +-
.../linux/runtime/docker/DockerClient.java | 7 +-
.../runtime/docker/DockerCommandExecutor.java | 7 +-
.../localizer/ContainerLocalizer.java | 7 +-
.../localizer/LocalResourcesTrackerImpl.java | 7 +-
.../localizer/LocalizedResource.java | 7 +-
.../localizer/ResourceLocalizationService.java | 9 +-
.../containermanager/localizer/ResourceSet.java | 7 +-
.../security/LocalizerSecurityInfo.java | 7 +-
.../security/LocalizerTokenSelector.java | 8 +-
.../sharedcache/SharedCacheUploadService.java | 8 +-
.../sharedcache/SharedCacheUploader.java | 7 +-
.../logaggregation/AppLogAggregatorImpl.java | 240 +--
.../logaggregation/LogAggregationService.java | 230 +--
.../SampleContainerLogAggregationPolicy.java | 8 +-
.../loghandler/NonAggregatingLogHandler.java | 8 +-
.../event/LogHandlerContainerFinishedEvent.java | 9 +-
.../monitor/ContainersMonitorImpl.java | 8 +-
.../scheduler/ContainerScheduler.java | 1 +
.../ConfigurationNodeLabelsProvider.java | 8 +-
.../ScriptBasedNodeLabelsProvider.java | 7 +-
.../recovery/NMLeveldbStateStoreService.java | 36 +-
.../recovery/NMNullStateStoreService.java | 2 +-
.../recovery/NMStateStoreService.java | 13 +-
.../security/NMContainerTokenSecretManager.java | 8 +-
.../security/NMTokenSecretManagerInNM.java | 8 +-
.../security/authorize/NMPolicyProvider.java | 22 +-
.../timelineservice/NMTimelinePublisher.java | 59 +-
.../util/CgroupsLCEResourcesHandler.java | 10 +-
.../util/DefaultLCEResourcesHandler.java | 8 +-
.../util/NodeManagerHardwareUtils.java | 8 +-
.../nodemanager/util/ProcessIdFileReader.java | 7 +-
.../nodemanager/webapp/NMWebServices.java | 17 +-
.../server/nodemanager/webapp/NavBlock.java | 8 +-
.../server/nodemanager/webapp/WebServer.java | 7 +-
.../impl/container-executor.c | 48 +-
.../impl/container-executor.h | 2 +
.../main/native/container-executor/impl/main.c | 13 +-
.../impl/modules/cgroups/cgroups-operations.c | 161 ++
.../impl/modules/cgroups/cgroups-operations.h | 55 +
.../impl/modules/common/constants.h | 29 +
.../impl/modules/common/module-configs.c | 41 +
.../impl/modules/common/module-configs.h | 33 +
.../impl/modules/gpu/gpu-module.c | 229 +++
.../impl/modules/gpu/gpu-module.h | 45 +
.../main/native/container-executor/impl/util.h | 7 +
.../container-executor/impl/utils/path-utils.c | 52 +
.../container-executor/impl/utils/path-utils.h | 35 +
.../impl/utils/string-utils.c | 109 +-
.../impl/utils/string-utils.h | 7 +-
.../test/modules/cgroups/test-cgroups-module.cc | 121 ++
.../test/modules/gpu/test-gpu-module.cc | 203 ++
.../test/test-container-executor-common.h | 36 +
.../test/test-container-executor.c | 23 +-
.../native/container-executor/test/test_main.cc | 11 +-
.../test/utils/test-path-utils.cc | 67 +
.../test/utils/test-string-utils.cc | 93 +
.../nodemanager/DummyContainerManager.java | 8 +-
.../nodemanager/MockNodeStatusUpdater.java | 7 +-
.../TestContainerManagerWithLCE.java | 8 +-
.../nodemanager/TestDirectoryCollection.java | 23 +
.../nodemanager/TestLinuxContainerExecutor.java | 8 +-
.../TestLinuxContainerExecutorWithMocks.java | 8 +-
.../nodemanager/TestNodeHealthService.java | 8 +-
.../nodemanager/TestNodeManagerReboot.java | 7 +-
.../nodemanager/TestNodeManagerResync.java | 8 +-
.../nodemanager/TestNodeStatusUpdater.java | 7 +-
.../amrmproxy/BaseAMRMProxyTest.java | 17 +-
.../amrmproxy/TestAMRMProxyService.java | 8 +-
.../BaseContainerManagerTest.java | 8 +-
.../containermanager/TestAuxServices.java | 7 +-
.../containermanager/TestContainerManager.java | 4 +-
.../application/TestApplication.java | 2 +-
.../launcher/TestContainerLaunch.java | 105 +
.../TestPrivilegedOperationExecutor.java | 8 +-
.../linux/resources/TestCGroupsHandlerImpl.java | 8 +-
.../resources/TestResourceHandlerModule.java | 8 +-
.../TestTrafficControlBandwidthHandlerImpl.java | 8 +-
.../linux/resources/TestTrafficController.java | 7 +-
.../runtime/TestDockerContainerRuntime.java | 8 +-
.../localizer/TestContainerLocalizer.java | 11 +-
.../TestAppLogAggregatorImpl.java | 25 +-
.../TestLogAggregationService.java | 244 ++-
.../TestNonAggregatingLogHandler.java | 10 +-
.../monitor/TestContainersMonitor.java | 4 +-
.../TestContainerSchedulerQueuing.java | 4 +-
.../recovery/NMMemoryStateStoreService.java | 4 +-
.../TestNMLeveldbStateStoreService.java | 6 +-
.../nodemanager/webapp/MockContainer.java | 4 +
.../nodemanager/webapp/TestNMWebServer.java | 4 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 3 +
.../ApplicationMasterService.java | 38 +-
.../server/resourcemanager/ClientRMService.java | 30 +-
.../resourcemanager/DefaultAMSProcessor.java | 8 +-
...pportunisticContainerAllocatorAMService.java | 2 +
.../resourcemanager/ResourceTrackerService.java | 90 +-
.../metrics/TimelineServiceV2Publisher.java | 12 +-
.../monitor/SchedulingMonitor.java | 8 +-
.../FifoIntraQueuePreemptionPlugin.java | 26 +-
.../recovery/ZKRMStateStore.java | 158 +-
.../server/resourcemanager/rmapp/RMApp.java | 35 +-
.../resourcemanager/rmapp/RMAppEventType.java | 3 -
.../server/resourcemanager/rmapp/RMAppImpl.java | 31 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 139 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 19 +-
.../rmcontainer/RMContainerImpl.java | 8 +-
.../server/resourcemanager/rmnode/RMNode.java | 6 +-
.../rmnode/RMNodeDecreaseContainerEvent.java | 39 -
.../resourcemanager/rmnode/RMNodeEventType.java | 2 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 29 +-
.../rmnode/RMNodeUpdateContainerEvent.java | 44 +
.../scheduler/AbstractYarnScheduler.java | 23 +-
.../scheduler/SchedulerApplicationAttempt.java | 51 +-
.../scheduler/capacity/CapacityScheduler.java | 12 +
.../distributed/NodeQueueLoadMonitor.java | 4 +
.../scheduler/event/ReleaseContainerEvent.java | 46 +
.../scheduler/event/SchedulerEventType.java | 3 +
.../scheduler/fair/FSAppAttempt.java | 22 +-
.../scheduler/fair/FairScheduler.java | 13 +
.../scheduler/fifo/FifoScheduler.java | 15 +-
.../webapp/CapacitySchedulerPage.java | 7 +-
.../resourcemanager/webapp/RMWebServices.java | 2 +-
.../resourcemanager/webapp/dao/AppInfo.java | 184 +-
.../resourcemanager/webapp/dao/AppsInfo.java | 4 +
.../webapp/dao/FairSchedulerQueueInfo.java | 18 -
.../yarn/server/resourcemanager/MockAM.java | 31 +-
.../yarn/server/resourcemanager/MockNM.java | 16 +
.../yarn/server/resourcemanager/MockNodes.java | 4 +-
.../yarn/server/resourcemanager/MockRM.java | 11 +
.../TestApplicationMasterService.java | 37 +-
.../resourcemanager/TestClientRMService.java | 14 +-
...pportunisticContainerAllocatorAMService.java | 200 +-
.../TestRMHATimelineCollectors.java | 126 ++
.../server/resourcemanager/TestRMRestart.java | 131 ++
.../TestResourceTrackerService.java | 32 +-
.../TestTokenClientRMService.java | 3 +
.../applicationsmanager/MockAsm.java | 17 +-
.../TestSystemMetricsPublisherForV2.java | 13 +-
.../server/resourcemanager/rmapp/MockRMApp.java | 18 +-
.../rmapp/attempt/TestBoundedAppender.java | 116 --
.../TestRMAppAttemptImplDiagnostics.java | 3 +-
.../attempt/TestRMAppAttemptTransitions.java | 32 +-
.../capacity/TestContainerResizing.java | 38 +-
.../capacity/TestIncreaseAllocationExpirer.java | 31 +-
.../security/TestDelegationTokenRenewer.java | 18 +
.../security/TestRMDelegationTokens.java | 4 +
.../webapp/TestRMWebServices.java | 5 +-
.../yarn/server/router/RouterMetrics.java | 236 +++
.../clientrm/FederationClientInterceptor.java | 37 +-
.../webapp/FederationInterceptorREST.java | 234 ++-
.../router/webapp/RouterWebServiceUtil.java | 109 +-
.../yarn/server/router/TestRouterMetrics.java | 298 +++
.../MockDefaultRequestInterceptorREST.java | 49 +-
.../webapp/TestFederationInterceptorREST.java | 29 +-
.../TestFederationInterceptorRESTRetry.java | 45 +
.../router/webapp/TestRouterWebServiceUtil.java | 311 +++
.../hadoop-yarn-server-tests/pom.xml | 11 +
.../hadoop/yarn/server/TestRMNMSecretKeys.java | 34 +-
.../TestTimelineServiceClientIntegration.java | 15 +-
.../security/TestTimelineAuthFilterForV2.java | 478 +++++
.../src/test/resources/krb5.conf | 28 -
.../AbstractTimelineReaderHBaseTestBase.java | 177 ++
...stTimelineReaderWebServicesHBaseStorage.java | 1007 +++++++---
.../storage/DataGeneratorForTest.java | 423 ++--
.../storage/TestHBaseTimelineStorageApps.java | 442 +++--
.../TestHBaseTimelineStorageEntities.java | 544 ++++--
.../storage/TestHBaseTimelineStorageSchema.java | 135 ++
.../storage/flow/TestFlowDataGenerator.java | 28 +-
.../flow/TestHBaseStorageFlowActivity.java | 72 +-
.../storage/flow/TestHBaseStorageFlowRun.java | 237 ++-
.../flow/TestHBaseStorageFlowRunCompaction.java | 67 +-
.../reader/filter/TimelineFilterUtils.java | 17 +
.../storage/HBaseTimelineReaderImpl.java | 12 +-
.../storage/HBaseTimelineWriterImpl.java | 294 +--
.../storage/TimelineSchemaCreator.java | 52 +-
.../storage/application/ApplicationColumn.java | 48 -
.../application/ApplicationColumnPrefix.java | 52 -
.../storage/application/ApplicationRowKey.java | 49 +-
.../storage/application/ApplicationTable.java | 2 +-
.../storage/apptoflow/AppToFlowColumn.java | 47 -
.../apptoflow/AppToFlowColumnPrefix.java | 206 ++
.../storage/apptoflow/AppToFlowRowKey.java | 101 +-
.../storage/apptoflow/AppToFlowTable.java | 21 +-
.../storage/common/BaseTable.java | 37 +-
.../storage/common/ColumnHelper.java | 50 +-
.../common/HBaseTimelineStorageUtils.java | 124 +-
.../storage/common/KeyConverterToString.java | 38 +
.../storage/common/LongConverter.java | 2 +-
.../storage/entity/EntityColumn.java | 48 -
.../storage/entity/EntityColumnPrefix.java | 51 -
.../storage/entity/EntityRowKey.java | 100 +-
.../storage/entity/EntityRowKeyPrefix.java | 11 +-
.../storage/entity/EntityTable.java | 4 +-
.../storage/flow/FlowActivityColumnPrefix.java | 58 +-
.../storage/flow/FlowActivityRowKey.java | 59 +-
.../storage/flow/FlowRunColumn.java | 53 +-
.../storage/flow/FlowRunColumnPrefix.java | 53 +-
.../storage/flow/FlowRunCoprocessor.java | 36 +-
.../storage/flow/FlowRunRowKey.java | 47 +-
.../storage/flow/FlowRunTable.java | 13 +-
.../timelineservice/storage/package-info.java | 6 +-
.../reader/AbstractTimelineStorageReader.java | 158 ++
.../storage/reader/ApplicationEntityReader.java | 77 +-
.../storage/reader/EntityTypeReader.java | 179 ++
.../reader/FlowActivityEntityReader.java | 30 +-
.../storage/reader/FlowRunEntityReader.java | 53 +-
.../storage/reader/GenericEntityReader.java | 201 +-
.../reader/SubApplicationEntityReader.java | 488 +++++
.../storage/reader/TimelineEntityReader.java | 60 +-
.../reader/TimelineEntityReaderFactory.java | 18 +-
.../subapplication/SubApplicationColumn.java | 108 +
.../SubApplicationColumnFamily.java | 68 +
.../SubApplicationColumnPrefix.java | 250 +++
.../subapplication/SubApplicationRowKey.java | 290 +++
.../SubApplicationRowKeyPrefix.java | 69 +
.../subapplication/SubApplicationTable.java | 174 ++
.../storage/subapplication/package-info.java | 28 +
.../common/TestHBaseTimelineStorageUtils.java | 33 +
.../storage/common/TestKeyConverters.java | 4 +
.../storage/common/TestRowKeys.java | 54 +-
.../storage/common/TestRowKeysAsString.java | 144 ++
.../collector/AppLevelTimelineCollector.java | 143 +-
.../AppLevelTimelineCollectorWithAgg.java | 150 ++
.../collector/NodeTimelineCollectorManager.java | 275 ++-
.../PerNodeTimelineCollectorsAuxService.java | 12 +-
.../collector/TimelineCollector.java | 18 +-
.../collector/TimelineCollectorManager.java | 14 +-
.../reader/TimelineDataToRetrieve.java | 35 +-
.../reader/TimelineEntityFilters.java | 160 +-
.../reader/TimelineReaderContext.java | 37 +-
.../reader/TimelineReaderManager.java | 40 +-
.../reader/TimelineReaderServer.java | 77 +-
.../reader/TimelineReaderUtils.java | 31 +-
.../reader/TimelineReaderWebServices.java | 693 ++++++-
.../reader/TimelineReaderWebServicesUtils.java | 98 +-
.../reader/TimelineUIDConverter.java | 84 +-
...neReaderAuthenticationFilterInitializer.java | 53 +
...elineReaderWhitelistAuthorizationFilter.java | 123 ++
...WhitelistAuthorizationFilterInitializer.java | 66 +
.../reader/security/package-info.java | 25 +
.../CollectorNodemanagerSecurityInfo.java | 69 +
...neV2DelegationTokenSecretManagerService.java | 126 ++
.../timelineservice/security/package-info.java | 25 +
.../storage/FileSystemTimelineReaderImpl.java | 21 +
.../storage/FileSystemTimelineWriterImpl.java | 15 +-
.../timelineservice/storage/TimelineReader.java | 23 +-
.../timelineservice/storage/TimelineWriter.java | 28 +-
.../storage/common/TimelineStorageUtils.java | 1 -
.../org.apache.hadoop.security.SecurityInfo | 14 +
.../TestNMTimelineCollectorManager.java | 4 +-
.../collector/TestTimelineCollector.java | 12 +-
.../reader/TestTimelineReaderWebServices.java | 4 +-
...elineReaderWhitelistAuthorizationFilter.java | 380 ++++
.../reader/TestTimelineUIDConverter.java | 17 +-
.../TestFileSystemTimelineReaderImpl.java | 115 +-
.../TestFileSystemTimelineWriterImpl.java | 8 +-
.../amfilter/TestAmFilterInitializer.java | 15 +-
.../src/site/markdown/Federation.md | 56 +-
.../src/site/markdown/TimelineServiceV2.md | 433 +++-
.../src/site/markdown/YarnCommands.md | 4 +-
.../src/main/webapp/app/adapters/yarn-queue.js | 30 -
.../app/adapters/yarn-queue/capacity-queue.js | 23 +
.../app/adapters/yarn-queue/fair-queue.js | 23 +
.../app/adapters/yarn-queue/fifo-queue.js | 23 +
.../app/adapters/yarn-queue/yarn-queue.js | 30 +
.../main/webapp/app/components/tree-selector.js | 19 +-
.../main/webapp/app/models/cluster-metric.js | 9 +-
.../src/main/webapp/app/models/yarn-queue.js | 94 -
.../app/models/yarn-queue/capacity-queue.js | 95 +
.../webapp/app/models/yarn-queue/fair-queue.js | 79 +
.../webapp/app/models/yarn-queue/fifo-queue.js | 52 +
.../webapp/app/models/yarn-queue/yarn-queue.js | 23 +
.../src/main/webapp/app/models/yarn-rm-node.js | 2 +-
.../main/webapp/app/routes/cluster-overview.js | 4 +-
.../src/main/webapp/app/routes/yarn-queue.js | 26 +-
.../src/main/webapp/app/routes/yarn-queues.js | 12 +-
.../main/webapp/app/routes/yarn-queues/index.js | 25 -
.../app/routes/yarn-queues/queues-selector.js | 25 -
.../main/webapp/app/serializers/yarn-queue.js | 129 --
.../serializers/yarn-queue/capacity-queue.js | 128 ++
.../app/serializers/yarn-queue/fair-queue.js | 92 +
.../app/serializers/yarn-queue/fifo-queue.js | 59 +
.../app/serializers/yarn-queue/yarn-queue.js | 47 +
.../webapp/app/templates/cluster-overview.hbs | 33 +-
.../components/queue-configuration-table.hbs | 54 -
.../templates/components/queue-navigator.hbs | 7 +-
.../yarn-queue/capacity-queue-conf-table.hbs | 54 +
.../yarn-queue/capacity-queue-info.hbs | 84 +
.../components/yarn-queue/capacity-queue.hbs | 63 +
.../yarn-queue/fair-queue-conf-table.hbs | 52 +
.../components/yarn-queue/fair-queue-info.hbs | 66 +
.../components/yarn-queue/fair-queue.hbs | 63 +
.../yarn-queue/fifo-queue-conf-table.hbs | 56 +
.../components/yarn-queue/fifo-queue-info.hbs | 47 +
.../components/yarn-queue/fifo-queue.hbs | 48 +
.../main/webapp/app/templates/yarn-nodes.hbs | 2 +-
.../webapp/app/templates/yarn-queue/info.hbs | 73 +-
.../main/webapp/app/templates/yarn-queues.hbs | 54 +-
.../src/main/webapp/app/utils/color-utils.js | 1 -
start-build-env.sh | 2 +
673 files changed, 33269 insertions(+), 8146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e5f54f/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e5f54f/hadoop-project/pom.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org