You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/10/05 22:07:18 UTC
[49/50] [abbrv] hadoop git commit: Merge branch 'trunk' into
HDFS-12943
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb75861e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb75861e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb75861e
Branch: refs/heads/HDFS-12943
Commit: eb75861eab63e1a56419b954ea12bf1b6f3bf8f7
Parents: 967aab6 cdf5d58
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Fri Oct 5 15:01:08 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Fri Oct 5 15:01:08 2018 -0700
----------------------------------------------------------------------
hadoop-assemblies/pom.xml | 4 +-
hadoop-build-tools/pom.xml | 2 +-
hadoop-client-modules/hadoop-client-api/pom.xml | 4 +-
.../hadoop-client-check-invariants/pom.xml | 4 +-
.../hadoop-client-check-test-invariants/pom.xml | 4 +-
.../hadoop-client-integration-tests/pom.xml | 4 +-
.../hadoop-client-minicluster/pom.xml | 4 +-
.../hadoop-client-runtime/pom.xml | 4 +-
hadoop-client-modules/hadoop-client/pom.xml | 4 +-
hadoop-client-modules/pom.xml | 2 +-
.../hadoop-cloud-storage/pom.xml | 4 +-
hadoop-cloud-storage-project/pom.xml | 4 +-
.../hadoop-annotations/pom.xml | 4 +-
.../hadoop-auth-examples/pom.xml | 4 +-
hadoop-common-project/hadoop-auth/pom.xml | 4 +-
hadoop-common-project/hadoop-common/pom.xml | 83 +-
.../hadoop/fs/FileSystemMultipartUploader.java | 25 +-
.../apache/hadoop/security/SecurityUtil.java | 10 +-
.../io/compress/zstd/ZStandardDecompressor.c | 2 +-
.../src/main/resources/core-default.xml | 10 +
.../src/site/markdown/SecureMode.md | 2 +-
.../AbstractContractMultipartUploaderTest.java | 22 +
.../hadoop/fs/sftp/TestSFTPFileSystem.java | 3 +
hadoop-common-project/hadoop-kms/pom.xml | 4 +-
.../crypto/key/kms/server/KMSJSONReader.java | 4 +-
hadoop-common-project/hadoop-minikdc/pom.xml | 4 +-
hadoop-common-project/hadoop-nfs/pom.xml | 4 +-
hadoop-common-project/pom.xml | 4 +-
hadoop-dist/pom.xml | 4 +-
.../hadoop/hdds/scm/XceiverClientGrpc.java | 23 -
.../hadoop/hdds/scm/XceiverClientRatis.java | 32 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../org/apache/hadoop/hdds/cli/GenericCli.java | 2 +-
.../hadoop/hdds/scm/XceiverClientSpi.java | 10 +-
.../scm/container/common/helpers/Pipeline.java | 4 +
.../StorageContainerLocationProtocol.java | 16 +
...rLocationProtocolClientSideTranslatorPB.java | 42 +
.../apache/hadoop/ozone/OzoneConfigKeys.java | 11 -
.../org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../apache/hadoop/ozone/audit/AuditLogger.java | 44 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 77 +-
.../org/apache/hadoop/ozone/audit/Auditor.java | 33 +
.../apache/hadoop/ozone/audit/package-info.java | 16 +-
...rLocationProtocolServerSideTranslatorPB.java | 32 +
.../StorageContainerLocationProtocol.proto | 27 +
.../common/src/main/resources/ozone-default.xml | 87 +-
.../ozone/audit/TestOzoneAuditLogger.java | 89 +-
.../common/src/test/resources/log4j2.properties | 4 +-
.../report/CommandStatusReportPublisher.java | 2 +-
.../common/statemachine/StateContext.java | 58 +-
.../CloseContainerCommandHandler.java | 14 +-
.../commandhandler/CommandHandler.java | 7 +-
.../DeleteBlocksCommandHandler.java | 43 +-
.../ReplicateContainerCommandHandler.java | 3 +-
.../states/endpoint/HeartbeatEndpointTask.java | 33 +-
.../states/endpoint/VersionEndpointTask.java | 44 +-
.../container/common/utils/HddsVolumeUtil.java | 2 +-
.../container/common/volume/VolumeSet.java | 178 ++--
.../container/keyvalue/KeyValueContainer.java | 6 +-
.../container/keyvalue/KeyValueHandler.java | 4 +-
.../container/ozoneimpl/ContainerReader.java | 20 +
.../container/ozoneimpl/OzoneContainer.java | 9 -
.../StorageContainerDatanodeProtocol.java | 13 -
.../protocol/commands/CommandForDatanode.java | 8 +-
.../ozone/protocol/commands/CommandStatus.java | 41 +-
.../commands/DeleteBlockCommandStatus.java | 92 ++
...rDatanodeProtocolClientSideTranslatorPB.java | 17 -
...rDatanodeProtocolServerSideTranslatorPB.java | 16 -
.../StorageContainerDatanodeProtocol.proto | 11 +-
.../ozone/container/common/ScmTestMock.java | 24 +-
.../common/report/TestReportPublisher.java | 4 +-
.../endpoint/TestHeartbeatEndpointTask.java | 12 +-
hadoop-hdds/pom.xml | 2 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 13 +-
.../block/DatanodeDeletedBlockTransactions.java | 30 +-
.../hdds/scm/block/DeletedBlockLogImpl.java | 32 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 17 +-
.../scm/command/CommandStatusReportHandler.java | 10 +-
.../container/CloseContainerEventHandler.java | 4 +-
.../scm/container/CloseContainerWatcher.java | 4 +-
.../hdds/scm/container/ContainerManager.java | 142 +++
.../hdds/scm/container/ContainerMapping.java | 698 -------------
.../scm/container/ContainerReportHandler.java | 28 +-
.../scm/container/ContainerStateManager.java | 51 +-
.../hadoop/hdds/scm/container/Mapping.java | 141 ---
.../hdds/scm/container/SCMContainerManager.java | 731 ++++++++++++++
.../placement/algorithms/SCMCommonPolicy.java | 4 +-
.../placement/metrics/SCMNodeMetric.java | 1 +
.../hadoop/hdds/scm/events/SCMEvents.java | 19 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 30 +-
.../hadoop/hdds/scm/node/NewNodeHandler.java | 9 +-
.../hadoop/hdds/scm/node/NodeManager.java | 91 +-
.../hadoop/hdds/scm/node/NodeManagerMXBean.java | 19 -
.../hadoop/hdds/scm/node/NodeStateManager.java | 142 ++-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 243 +++--
.../hdds/scm/node/SCMNodeStorageStatMap.java | 2 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 9 +-
.../hdds/scm/node/states/Node2ContainerMap.java | 2 +-
.../hdds/scm/node/states/Node2PipelineMap.java | 75 ++
.../hdds/scm/node/states/NodeStateMap.java | 68 +-
.../hdds/scm/pipelines/Node2PipelineMap.java | 75 --
.../hdds/scm/pipelines/PipelineSelector.java | 18 +-
.../hdds/scm/server/SCMChillModeManager.java | 71 +-
.../scm/server/SCMClientProtocolServer.java | 44 +-
.../server/SCMDatanodeHeartbeatDispatcher.java | 11 +-
.../scm/server/SCMDatanodeProtocolServer.java | 24 +-
.../hadoop/hdds/scm/server/SCMMXBean.java | 12 +
.../scm/server/StorageContainerManager.java | 79 +-
.../commands/RetriableDatanodeEventWatcher.java | 58 ++
.../ozone/protocol/commands/package-info.java | 18 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../command/TestCommandStatusReportHandler.java | 12 +-
.../hdds/scm/container/MockNodeManager.java | 169 ++--
.../TestCloseContainerEventHandler.java | 4 +-
.../scm/container/TestContainerMapping.java | 380 -------
.../container/TestContainerReportHandler.java | 22 +-
.../container/TestContainerStateManager.java | 2 +-
.../scm/container/TestSCMContainerManager.java | 378 +++++++
.../hdds/scm/node/TestContainerPlacement.java | 15 +-
.../hdds/scm/node/TestDeadNodeHandler.java | 145 ++-
.../hadoop/hdds/scm/node/TestNodeManager.java | 123 +--
.../hdds/scm/node/TestNodeReportHandler.java | 5 +-
.../scm/server/TestSCMChillModeManager.java | 41 +-
.../TestSCMDatanodeHeartbeatDispatcher.java | 2 +-
.../container/TestCloseContainerWatcher.java | 12 +-
.../testutils/ReplicationNodeManagerMock.java | 150 +--
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +-
.../main/java/org/apache/hadoop/fs/XAttr.java | 2 +-
.../org/apache/hadoop/hdfs/AddBlockFlag.java | 8 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 6 +-
.../hadoop/hdfs/DFSInotifyEventInputStream.java | 2 +-
.../java/org/apache/hadoop/hdfs/DFSPacket.java | 16 +-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 4 +-
.../hadoop/hdfs/DistributedFileSystem.java | 13 +-
.../hadoop/hdfs/NameNodeProxiesClient.java | 4 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 12 +-
.../hadoop/hdfs/client/impl/LeaseRenewer.java | 2 +-
.../hdfs/protocol/ClientDatanodeProtocol.java | 2 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 30 +-
.../hdfs/protocol/ReconfigurationProtocol.java | 2 +-
.../token/block/BlockTokenIdentifier.java | 4 +-
.../hdfs/shortcircuit/DfsClientShmManager.java | 5 +-
.../hadoop/hdfs/util/ByteArrayManager.java | 4 +-
.../hadoop/hdfs/util/StripedBlockUtil.java | 59 +-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 5 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 1 -
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 17 +-
.../hadoop/fs/http/server/FSOperations.java | 50 +
.../http/server/HttpFSParametersProvider.java | 3 +
.../hadoop/fs/http/server/HttpFSServer.java | 13 +
.../fs/http/client/BaseTestHttpFSWith.java | 107 +-
.../hadoop/fs/http/server/TestHttpFSServer.java | 96 ++
.../hadoop-hdfs-native-client/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 4 +-
...uterAdminProtocolServerSideTranslatorPB.java | 2 +-
.../resolver/ActiveNamenodeResolver.java | 11 +-
.../resolver/FileSubclusterResolver.java | 1 +
.../federation/resolver/MountTableResolver.java | 2 +
.../MultipleDestinationMountTableResolver.java | 4 +-
.../resolver/NamenodeStatusReport.java | 5 +-
.../resolver/order/AvailableSpaceResolver.java | 2 +-
.../resolver/order/LocalResolver.java | 2 +-
.../federation/resolver/package-info.java | 5 +-
.../federation/router/ConnectionPool.java | 3 +-
.../federation/router/NameserviceManager.java | 9 +
.../hdfs/server/federation/router/Quota.java | 4 +-
.../hdfs/server/federation/router/Router.java | 5 +-
.../federation/router/RouterAdminServer.java | 4 +-
.../federation/router/RouterClientProtocol.java | 6 +-
.../federation/router/RouterQuotaManager.java | 6 +-
.../federation/router/RouterQuotaUsage.java | 4 +-
.../federation/router/RouterRpcClient.java | 23 +-
.../federation/router/RouterRpcServer.java | 14 +-
.../router/RouterSafemodeService.java | 26 +-
.../federation/router/RouterStateManager.java | 9 +
.../federation/store/CachedRecordStore.java | 4 +-
.../federation/store/MembershipStore.java | 16 +-
.../server/federation/store/RouterStore.java | 10 +-
.../federation/store/StateStoreService.java | 19 +-
.../federation/store/StateStoreUtils.java | 4 +
.../driver/StateStoreRecordOperations.java | 15 +-
.../store/driver/impl/StateStoreBaseImpl.java | 1 -
.../server/federation/store/package-info.java | 12 +-
.../federation/store/records/BaseRecord.java | 4 +-
.../store/records/MembershipState.java | 3 -
.../federation/store/records/MountTable.java | 26 +-
.../server/federation/store/records/Query.java | 2 +-
.../federation/utils/ConsistentHashRing.java | 2 +-
.../hdfs/tools/federation/RouterAdmin.java | 5 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +-
.../server/blockmanagement/BlockManager.java | 19 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 4 +-
.../hdfs/server/datanode/DirectoryScanner.java | 617 ++++++------
.../server/datanode/fsdataset/FsVolumeSpi.java | 36 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 177 +++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 14 +
.../datanode/fsdataset/impl/FsDatasetUtil.java | 30 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 203 ++--
.../datanode/fsdataset/impl/FsVolumeList.java | 5 +-
.../fsdataset/impl/ProvidedVolumeImpl.java | 50 +-
.../datanode/fsdataset/impl/ReplicaMap.java | 25 +
.../namenode/ErasureCodingPolicyManager.java | 6 +
.../hdfs/server/namenode/FSNamesystem.java | 9 +
.../namenode/metrics/ECBlockGroupsMBean.java | 5 +
.../src/main/resources/hdfs-default.xml | 9 +
.../src/main/webapps/hdfs/dfshealth.html | 4 +-
.../src/site/markdown/HdfsPermissionsGuide.md | 2 +
.../src/site/markdown/HdfsProvidedStorage.md | 2 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 5 +
.../TestRBWBlockInvalidation.java | 38 +
.../server/datanode/SimulatedFSDataset.java | 222 ++---
.../TestDataNodeMultipleRegistrations.java | 2 +-
.../server/datanode/TestDirectoryScanner.java | 249 +++--
.../datanode/extdataset/ExternalVolumeImpl.java | 10 +-
.../fsdataset/impl/TestFsVolumeList.java | 64 +-
.../fsdataset/impl/TestProvidedImpl.java | 13 +-
.../server/namenode/TestAddStripedBlocks.java | 4 +-
.../namenode/TestListCorruptFileBlocks.java | 50 +-
.../server/namenode/TestNameNodeMXBean.java | 49 +
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 8 +-
hadoop-hdfs-project/pom.xml | 4 +-
.../hadoop-mapreduce-client-app/pom.xml | 4 +-
.../hadoop-mapreduce-client-common/pom.xml | 4 +-
.../hadoop-mapreduce-client-core/pom.xml | 4 +-
.../hadoop/mapreduce/JobResourceUploader.java | 2 +-
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +-
.../hadoop-mapreduce-client-hs/pom.xml | 4 +-
.../hadoop-mapreduce-client-jobclient/pom.xml | 4 +-
.../hadoop-mapreduce-client-nativetask/pom.xml | 24 +-
.../hadoop-mapreduce-client-shuffle/pom.xml | 4 +-
.../hadoop-mapreduce-client-uploader/pom.xml | 4 +-
.../hadoop-mapreduce-client/pom.xml | 4 +-
.../hadoop-mapreduce-examples/pom.xml | 4 +-
hadoop-mapreduce-project/pom.xml | 4 +-
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-minicluster/pom.xml | 4 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 12 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 24 +-
.../apache/hadoop/ozone/client/OzoneClient.java | 7 +
.../apache/hadoop/ozone/client/OzoneVolume.java | 32 +-
hadoop-ozone/common/src/main/bin/ozone | 7 +-
.../dist/dev-support/bin/dist-layout-stitching | 2 +-
hadoop-ozone/dist/pom.xml | 11 +
.../main/compose/ozones3/docker-compose.yaml | 58 ++
.../dist/src/main/compose/ozones3/docker-config | 73 ++
.../src/main/smoketest/ozonefs/ozonefs.robot | 75 +-
.../dist/src/main/smoketest/s3/awscli.robot | 63 ++
hadoop-ozone/dist/src/main/smoketest/test.sh | 8 +-
hadoop-ozone/docs/content/Settings.md | 2 +-
hadoop-ozone/integration-test/pom.xml | 4 +
.../TestContainerStateManagerIntegration.java | 53 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 23 +-
.../hdds/scm/pipeline/TestNodeFailure.java | 18 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 30 +-
.../hdds/scm/pipeline/TestSCMRestart.java | 33 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 4 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 6 +-
.../ozone/TestOzoneConfigurationFields.java | 4 +-
.../ozone/TestStorageContainerManager.java | 154 +--
.../ozone/client/rest/TestOzoneRestClient.java | 2 +-
.../rpc/TestCloseContainerHandlingByClient.java | 6 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 43 +-
.../common/impl/TestContainerPersistence.java | 3 +-
.../commandhandler/TestBlockDeletion.java | 84 +-
.../TestCloseContainerByPipeline.java | 6 +-
.../TestCloseContainerHandler.java | 2 +-
.../hadoop/ozone/om/TestScmChillMode.java | 330 ++++--
.../hadoop/ozone/ozShell/TestOzoneShell.java | 55 +-
.../apache/hadoop/ozone/scm/TestSCMMXBean.java | 38 +-
.../hadoop/ozone/web/client/TestBuckets.java | 4 +-
.../hadoop/ozone/om/OMMetadataManager.java | 6 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 14 +
.../apache/hadoop/ozone/om/OzoneManager.java | 90 +-
.../hadoop/ozone/om/OzoneManagerLock.java | 58 +-
.../apache/hadoop/ozone/om/S3BucketManager.java | 59 ++
.../hadoop/ozone/om/S3BucketManagerImpl.java | 194 ++++
.../hadoop/ozone/om/exceptions/OMException.java | 5 +-
.../web/ozShell/bucket/CreateBucketHandler.java | 13 +-
.../web/ozShell/bucket/InfoBucketHandler.java | 14 +-
.../web/ozShell/bucket/ListBucketHandler.java | 15 +-
.../ozone/web/ozShell/keys/ListKeyHandler.java | 16 +-
.../web/ozShell/volume/CreateVolumeHandler.java | 21 +-
.../web/ozShell/volume/InfoVolumeHandler.java | 16 +-
.../web/ozShell/volume/ListVolumeHandler.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerLock.java | 3 +-
.../hadoop/ozone/om/TestS3BucketManager.java | 115 +++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 +-
hadoop-ozone/pom.xml | 14 +-
hadoop-ozone/s3gateway/pom.xml | 178 ++++
.../CommonHeadersContainerResponseFilter.java | 39 +
.../apache/hadoop/ozone/s3/EndpointBase.java | 102 ++
.../org/apache/hadoop/ozone/s3/Gateway.java | 66 ++
.../hadoop/ozone/s3/GatewayApplication.java | 29 +
.../hadoop/ozone/s3/OzoneClientProducer.java | 48 +
.../ozone/s3/OzoneConfigurationHolder.java | 43 +
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 55 +
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 85 ++
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 143 +++
.../hadoop/ozone/s3/bucket/DeleteBucket.java | 44 +
.../hadoop/ozone/s3/bucket/HeadBucket.java | 67 ++
.../hadoop/ozone/s3/bucket/PutBucket.java | 44 +
.../hadoop/ozone/s3/bucket/package-info.java | 30 +
.../ozone/s3/commontypes/CommonPrefix.java | 47 +
.../ozone/s3/commontypes/IsoDateAdapter.java | 47 +
.../ozone/s3/commontypes/KeyMetadata.java | 87 ++
.../ozone/s3/commontypes/package-info.java | 29 +
.../hadoop/ozone/s3/exception/OS3Exception.java | 161 +++
.../ozone/s3/exception/OS3ExceptionMapper.java | 41 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 81 ++
.../hadoop/ozone/s3/exception/package-info.java | 21 +
.../hadoop/ozone/s3/object/DeleteObject.java | 51 +
.../hadoop/ozone/s3/object/HeadObject.java | 58 ++
.../hadoop/ozone/s3/object/ListObject.java | 119 +++
.../ozone/s3/object/ListObjectResponse.java | 147 +++
.../hadoop/ozone/s3/object/PutObject.java | 61 ++
.../hadoop/ozone/s3/object/package-info.java | 29 +
.../apache/hadoop/ozone/s3/package-info.java | 22 +
.../src/main/resources/META-INF/beans.xml | 20 +
.../webapps/s3gateway/WEB-INF/beans.xml | 20 +
.../resources/webapps/s3gateway/WEB-INF/web.xml | 36 +
.../hadoop/ozone/client/ObjectStoreStub.java | 110 ++
.../hadoop/ozone/client/OzoneBucketStub.java | 143 +++
.../hadoop/ozone/client/OzoneClientStub.java | 37 +
.../hadoop/ozone/client/OzoneVolumeStub.java | 95 ++
.../hadoop/ozone/client/package-info.java | 21 +
.../ozone/s3/TestVirtualHostStyleFilter.java | 163 +++
.../ozone/s3/bucket/TestBucketResponse.java | 40 +
.../hadoop/ozone/s3/bucket/TestGetBucket.java | 114 +++
.../hadoop/ozone/s3/bucket/TestHeadBucket.java | 95 ++
.../hadoop/ozone/s3/bucket/package-info.java | 21 +
.../ozone/s3/exception/TestOS3Exception.java | 49 +
.../hadoop/ozone/s3/exception/package-info.java | 22 +
.../ozone/s3/object/TestDeleteObject.java | 56 ++
.../hadoop/ozone/s3/object/package-info.java | 21 +
.../apache/hadoop/ozone/s3/package-info.java | 21 +
.../GenerateOzoneRequiredConfigurations.java | 130 +--
...TestGenerateOzoneRequiredConfigurations.java | 167 +++-
.../hadoop/ozone/scm/TestContainerSQLCli.java | 12 +-
hadoop-project-dist/pom.xml | 4 +-
hadoop-project/pom.xml | 44 +-
hadoop-project/src/site/site.xml | 6 +
hadoop-tools/hadoop-aliyun/pom.xml | 2 +-
.../aliyun/oss/AliyunOSSBlockOutputStream.java | 5 +-
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 5 +-
.../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 8 +-
.../oss/AssumedRoleCredentialProvider.java | 115 +++
.../apache/hadoop/fs/aliyun/oss/Constants.java | 22 +
.../site/markdown/tools/hadoop-aliyun/index.md | 50 +
.../fs/aliyun/oss/TestAliyunCredentials.java | 55 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 4 +-
hadoop-tools/hadoop-archives/pom.xml | 4 +-
hadoop-tools/hadoop-aws/pom.xml | 4 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 10 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 30 +-
.../hadoop/fs/s3a/s3guard/DDBPathMetadata.java | 16 +-
.../fs/s3a/s3guard/DirListingMetadata.java | 15 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 7 +-
.../fs/s3a/s3guard/ExpirableMetadata.java | 39 +
.../hadoop/fs/s3a/s3guard/PathMetadata.java | 2 +-
.../PathMetadataDynamoDBTranslation.java | 61 +-
.../apache/hadoop/fs/s3a/s3guard/S3Guard.java | 61 +-
.../site/markdown/tools/hadoop-aws/s3guard.md | 11 +
.../apache/hadoop/fs/s3a/ITestS3GuardTtl.java | 105 ++
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 13 +
.../hadoop/fs/s3a/auth/ITestAssumeRole.java | 47 +-
.../hadoop/fs/s3a/auth/RoleTestUtils.java | 6 -
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 16 +-
.../TestPathMetadataDynamoDBTranslation.java | 48 +-
.../hadoop/fs/s3a/s3guard/TestS3Guard.java | 6 +-
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
hadoop-tools/hadoop-azure/pom.xml | 42 +-
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 44 +
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 138 ++-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 39 +-
.../fs/azurebfs/SecureAzureBlobFileSystem.java | 2 +-
.../azurebfs/constants/ConfigurationKeys.java | 3 +
.../constants/FileSystemConfigurations.java | 2 +
.../exceptions/AbfsRestOperationException.java | 9 +
.../extensions/AbfsAuthorizationException.java | 41 +
.../fs/azurebfs/extensions/AbfsAuthorizer.java | 57 ++
.../azurebfs/oauth2/AzureADAuthenticator.java | 35 +-
.../fs/azurebfs/services/AbfsRestOperation.java | 9 +
.../azurebfs/AbstractAbfsIntegrationTest.java | 8 +-
.../ITestAzureBlobFileSystemAuthorization.java | 362 +++++++
.../ITestAzureBlobFileSystemBackCompat.java | 2 +-
.../ITestAzureBlobFileSystemFileStatus.java | 2 +-
.../ITestAzureBlobFileSystemPermission.java | 4 +-
.../ITestAzureBlobFileSystemRandomRead.java | 2 +-
.../azurebfs/ITestAzureBlobFilesystemAcl.java | 116 +--
.../fs/azurebfs/ITestClientUrlScheme.java | 101 ++
.../fs/azurebfs/ITestOauthOverAbfsScheme.java | 63 ++
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 8 +-
.../contract/AbfsFileSystemContract.java | 2 +-
.../ITestAbfsFileSystemContractDistCp.java | 3 +
.../azurebfs/extensions/MockAbfsAuthorizer.java | 87 ++
.../fs/azurebfs/extensions/package-info.java | 22 +
.../services/TestOauthFailOverHttp.java | 55 -
hadoop-tools/hadoop-datajoin/pom.xml | 4 +-
hadoop-tools/hadoop-distcp/pom.xml | 4 +-
hadoop-tools/hadoop-extras/pom.xml | 4 +-
hadoop-tools/hadoop-fs2img/pom.xml | 4 +-
hadoop-tools/hadoop-gridmix/pom.xml | 4 +-
hadoop-tools/hadoop-kafka/pom.xml | 4 +-
hadoop-tools/hadoop-openstack/pom.xml | 4 +-
hadoop-tools/hadoop-pipes/pom.xml | 4 +-
hadoop-tools/hadoop-resourceestimator/pom.xml | 2 +-
hadoop-tools/hadoop-rumen/pom.xml | 4 +-
hadoop-tools/hadoop-sls/pom.xml | 4 +-
hadoop-tools/hadoop-streaming/pom.xml | 4 +-
hadoop-tools/hadoop-tools-dist/pom.xml | 4 +-
hadoop-tools/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 22 +-
.../yarn/conf/TestYarnConfigurationFields.java | 2 +
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../yarn/service/client/ApiServiceClient.java | 20 +
.../hadoop/yarn/service/webapp/ApiServer.java | 34 +-
...RN-Simplified-V1-API-Layer-For-Services.yaml | 5 +
.../hadoop-yarn-services-core/pom.xml | 2 +-
.../hadoop/yarn/service/ClientAMProtocol.java | 5 +
.../hadoop/yarn/service/ClientAMService.java | 12 +
.../hadoop/yarn/service/ServiceEvent.java | 14 +-
.../hadoop/yarn/service/ServiceEventType.java | 3 +-
.../hadoop/yarn/service/ServiceManager.java | 332 +++++--
.../hadoop/yarn/service/ServiceScheduler.java | 2 +
.../service/api/records/ContainerState.java | 2 +-
.../yarn/service/api/records/Service.java | 15 +
.../yarn/service/api/records/ServiceState.java | 7 +-
.../yarn/service/client/ServiceClient.java | 21 +
.../yarn/service/component/Component.java | 275 +++--
.../yarn/service/component/ComponentEvent.java | 10 -
.../service/component/ComponentEventType.java | 1 +
.../yarn/service/component/ComponentState.java | 3 +-
.../component/instance/ComponentInstance.java | 269 ++++-
.../instance/ComponentInstanceEventType.java | 3 +-
.../instance/ComponentInstanceState.java | 3 +-
.../containerlaunch/ContainerLaunchService.java | 4 +-
.../pb/client/ClientAMProtocolPBClientImpl.java | 13 +
.../service/ClientAMProtocolPBServiceImpl.java | 13 +
.../yarn/service/provider/ProviderUtils.java | 9 +-
.../yarn/service/utils/ServiceApiUtil.java | 57 +-
.../yarn/service/utils/SliderFileSystem.java | 49 +
.../src/main/proto/ClientAMProtocol.proto | 8 +
.../yarn/service/MockRunningServiceContext.java | 20 +-
.../hadoop/yarn/service/ServiceTestUtils.java | 2 +-
.../hadoop/yarn/service/TestServiceManager.java | 136 ++-
.../yarn/service/TestYarnNativeServices.java | 46 +
.../yarn/service/client/TestServiceCLI.java | 17 +
.../yarn/service/component/TestComponent.java | 239 ++++-
.../instance/TestComponentInstance.java | 177 +++-
.../yarn/service/utils/TestServiceApiUtil.java | 39 +
.../src/test/resources/log4j.properties | 19 +
.../hadoop-yarn-services/pom.xml | 2 +-
.../hadoop-yarn-submarine/README.md | 6 +-
.../hadoop-yarn-submarine/pom.xml | 10 +-
.../base/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0 | 69 ++
.../base/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0 | 67 ++
.../src/main/docker/build-all.sh | 32 +
.../ubuntu-16.04/Dockerfile.cpu.tf_1.8.0 | 22 +
.../ubuntu-16.04/Dockerfile.gpu.tf_1.8.0 | 22 +
.../cifar10_estimator_tf_1.8.0/README.md | 542 ++++++++++
.../cifar10_estimator_tf_1.8.0/cifar10.py | 113 +++
.../cifar10_estimator_tf_1.8.0/cifar10_main.py | 521 ++++++++++
.../cifar10_estimator_tf_1.8.0/cifar10_model.py | 80 ++
.../cifar10_estimator_tf_1.8.0/cifar10_utils.py | 154 +++
.../generate_cifar10_tfrecords.py | 114 +++
.../cifar10_estimator_tf_1.8.0/model_base.py | 219 ++++
.../zeppelin-notebook-example/Dockerfile.gpu | 75 ++
.../zeppelin-notebook-example/run_container.sh | 22 +
.../docker/zeppelin-notebook-example/shiro.ini | 120 +++
.../zeppelin-notebook-example/zeppelin-site.xml | 569 +++++++++++
.../yarnservice/YarnServiceJobSubmitter.java | 16 +-
.../src/site/DeveloperGuide.md | 26 -
.../src/site/QuickStart.md | 134 ---
.../src/site/markdown/DeveloperGuide.md | 24 +
.../src/site/markdown/Examples.md | 21 +
.../src/site/markdown/Index.md | 42 +
.../src/site/markdown/QuickStart.md | 174 ++++
.../markdown/RunningDistributedCifar10TFJobs.md | 162 +++
.../src/site/markdown/RunningZeppelinOnYARN.md | 37 +
.../src/site/markdown/WriteDockerfile.md | 117 +++
.../src/site/resources/css/site.css | 29 +
.../src/site/resources/images/job-logs-ui.png | Bin 0 -> 229944 bytes
.../images/multiple-tensorboard-jobs.png | Bin 0 -> 184717 bytes
.../resources/images/tensorboard-service.png | Bin 0 -> 107567 bytes
.../hadoop-yarn-submarine/src/site/site.xml | 28 +
.../hadoop-yarn-applications/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +-
.../yarn/client/api/async/AMRMClientAsync.java | 11 +
.../api/async/impl/AMRMClientAsyncImpl.java | 9 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 12 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 2 +
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 +-
.../hadoop/yarn/client/AMRMClientUtils.java | 7 +
.../hadoop/yarn/client/api/AppAdminClient.java | 13 +
.../src/main/resources/yarn-default.xml | 18 +
.../client/api/impl/TestTimelineClient.java | 25 +-
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 +-
.../pom.xml | 4 +-
.../hadoop-yarn-server-common/pom.xml | 4 +-
.../yarn/server/AMHeartbeatRequestHandler.java | 23 +-
.../hadoop/yarn/server/AMRMClientRelayer.java | 25 +-
.../utils/FederationRegistryClient.java | 10 +-
.../yarn/server/uam/UnmanagedAMPoolManager.java | 15 +
.../server/uam/UnmanagedApplicationManager.java | 14 +
.../yarn/server/MockResourceManagerFacade.java | 72 +-
.../yarn/server/TestAMRMClientRelayer.java | 25 +
.../hadoop-yarn-server-nodemanager/pom.xml | 4 +-
.../amrmproxy/FederationInterceptor.java | 388 +++++---
.../containermanager/ContainerManagerImpl.java | 1 +
.../container/ContainerImpl.java | 3 +
.../launcher/ContainerLaunch.java | 1 +
.../resources/gpu/GpuResourceHandlerImpl.java | 3 +
.../runtime/DefaultLinuxContainerRuntime.java | 6 +-
.../runtime/DockerLinuxContainerRuntime.java | 23 +-
.../container-executor/impl/utils/docker-util.c | 4 +-
.../amrmproxy/TestFederationInterceptor.java | 298 +++---
.../TestableFederationInterceptor.java | 107 ++
.../containermanager/TestNMProxy.java | 2 +-
.../resources/gpu/TestGpuResourceHandler.java | 9 +-
.../runtime/TestDockerContainerRuntime.java | 91 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 +-
.../ApplicationMasterService.java | 15 +-
.../server/resourcemanager/RMAppManager.java | 11 +-
.../server/resourcemanager/RMServerUtils.java | 9 +-
.../resourcemanager/ResourceTrackerService.java | 23 +
.../resourcemanager/recovery/RMStateStore.java | 2 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 63 +-
.../scheduler/AbstractYarnScheduler.java | 21 +-
.../scheduler/ResourceLimits.java | 24 +
.../scheduler/SchedulerUtils.java | 62 +-
.../scheduler/YarnScheduler.java | 9 +-
.../scheduler/capacity/CapacityScheduler.java | 7 +-
.../scheduler/capacity/ParentQueue.java | 16 +-
.../allocator/RegularContainerAllocator.java | 18 +-
.../processor/PlacementConstraintProcessor.java | 12 +-
.../scheduler/fair/AllocationConfiguration.java | 13 +
.../scheduler/fair/FSLeafQueue.java | 10 +
.../scheduler/fair/FSParentQueue.java | 15 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 7 +
.../scheduler/fair/FairScheduler.java | 52 +-
.../allocation/AllocationFileQueueParser.java | 7 +
.../fair/allocation/QueueProperties.java | 16 +-
.../webapp/FairSchedulerPage.java | 4 +
.../webapp/dao/FairSchedulerQueueInfo.java | 9 +-
.../resourcemanager/AppManagerTestBase.java | 107 ++
.../ApplicationMasterServiceTestBase.java | 582 +++++++++++
.../resourcemanager/MockMemoryRMStateStore.java | 1 +
.../yarn/server/resourcemanager/MockRM.java | 48 +
.../server/resourcemanager/TestAppManager.java | 89 +-
.../TestAppManagerWithFairScheduler.java | 177 ++++
.../TestApplicationMasterService.java | 993 -------------------
.../TestApplicationMasterServiceCapacity.java | 207 ++++
.../TestApplicationMasterServiceFair.java | 118 +++
...TestApplicationMasterServiceInterceptor.java | 217 ++++
.../resourcemanager/TestClientRMService.java | 104 +-
.../server/resourcemanager/TestRMRestart.java | 68 ++
.../resourcemanager/TestRMServerUtils.java | 98 +-
.../TestResourceTrackerService.java | 45 +
.../rmapp/TestRMAppTransitions.java | 171 ++--
.../scheduler/AppMetricsChecker.java | 122 +++
.../scheduler/ResourceMetricsChecker.java | 170 ++++
.../scheduler/TestQueueMetrics.java | 550 +++++++---
.../scheduler/TestSchedulerUtils.java | 144 +--
.../capacity/TestCapacityScheduler.java | 39 +-
.../capacity/TestContainerAllocation.java | 78 ++
.../TestNodeLabelContainerAllocation.java | 62 ++
.../scheduler/fair/FairSchedulerTestBase.java | 4 +-
.../fair/TestAllocationFileLoaderService.java | 37 +-
.../TestApplicationMasterServiceWithFS.java | 174 ++++
.../scheduler/fair/TestFairScheduler.java | 140 ++-
.../allocationfile/AllocationFileQueue.java | 6 +-
.../AllocationFileQueueBuilder.java | 6 +
.../AllocationFileQueueProperties.java | 12 +
.../webapp/TestRMWebServicesAppAttempts.java | 397 ++++++++
...TestRMWebServicesAppCustomResourceTypes.java | 170 ++++
.../webapp/TestRMWebServicesApps.java | 287 ------
...estRMWebServicesAppsCustomResourceTypes.java | 81 +-
...RMWebServicesCustomResourceTypesCommons.java | 99 ++
.../webapp/TestRMWebServicesFairScheduler.java | 160 ---
...ustomResourceTypesConfigurationProvider.java | 4 +-
.../FairSchedulerJsonVerifications.java | 4 +-
.../FairSchedulerXmlVerifications.java | 4 +-
.../TestRMWebServicesFairScheduler.java | 160 +++
...ervicesFairSchedulerCustomResourceTypes.java | 4 +-
.../webapp/helper/AppInfoJsonVerifications.java | 4 +-
.../webapp/helper/AppInfoXmlVerifications.java | 4 +-
.../webapp/helper/BufferedClientResponse.java | 11 +-
.../helper/JsonCustomResourceTypeTestcase.java | 9 +
.../ResourceRequestsJsonVerifications.java | 4 +-
.../ResourceRequestsXmlVerifications.java | 4 +-
.../helper/XmlCustomResourceTypeTestCase.java | 9 +
.../hadoop-yarn-server-router/pom.xml | 4 +-
.../clientrm/FederationClientInterceptor.java | 2 +-
.../TestFederationClientInterceptor.java | 9 +
.../pom.xml | 4 +-
.../hadoop-yarn-server-tests/pom.xml | 4 +-
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../pom.xml | 2 +-
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../hadoop-yarn-server-timelineservice/pom.xml | 4 +-
.../collector/TimelineCollectorWebService.java | 16 +-
.../PerNodeAggTimelineCollectorMetrics.java | 117 +++
.../metrics/TimelineReaderMetrics.java | 113 +++
.../timelineservice/metrics/package-info.java | 28 +
.../reader/TimelineReaderWebServices.java | 166 +++-
.../TestPerNodeAggTimelineCollectorMetrics.java | 56 ++
.../reader/TestTimelineReaderMetrics.java | 56 ++
.../hadoop-yarn-server-web-proxy/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +-
.../src/site/markdown/DockerContainers.md | 73 +-
.../src/site/markdown/FairScheduler.md | 2 +
.../src/site/markdown/registry/yarn-registry.md | 2 +-
.../markdown/yarn-service/YarnServiceAPI.md | 2 +-
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 4 +-
.../main/webapp/app/adapters/yarn-service.js | 32 +
.../webapp/app/controllers/app-table-columns.js | 14 +-
.../main/webapp/app/controllers/application.js | 7 +-
.../src/main/webapp/app/controllers/yarn-app.js | 20 +-
.../src/main/webapp/app/models/yarn-app.js | 15 +-
.../src/main/webapp/app/models/yarn-service.js | 30 +
.../src/main/webapp/app/router.js | 7 +-
.../src/main/webapp/app/routes/application.js | 8 +-
.../src/main/webapp/app/routes/yarn-app.js | 12 +
.../src/main/webapp/app/serializers/yarn-app.js | 4 +-
.../main/webapp/app/serializers/yarn-service.js | 44 +
.../src/main/webapp/app/templates/error.hbs | 2 +-
.../src/main/webapp/app/templates/notfound.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-app.hbs | 28 +-
.../main/webapp/app/templates/yarn-services.hbs | 4 +-
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 +-
hadoop-yarn-project/pom.xml | 4 +-
pom.xml | 4 +-
645 files changed, 22630 insertions(+), 7630 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org