You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2019/10/14 15:20:11 UTC
[hadoop-ozone] 01/01: Merge remote-tracking branch 'origin/master'
into HDDS-1880-Decom
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 5547d850fb0d21e25648ce155b3287795399e068
Merge: c09e0a1 e722e4d
Author: Márton Elek <el...@apache.org>
AuthorDate: Mon Oct 14 16:42:02 2019 +0200
Merge remote-tracking branch 'origin/master' into HDDS-1880-Decom
BUILDING.txt | 511 +++++++++++++++++++
LICENSE.txt | 258 ++++++++++
NOTICE.txt | 34 ++
README.txt | 7 +
.../checks/build.sh => dev-support/bin/qbt | 7 +-
.../build.sh => dev-support/bin/smart-apply-patch | 7 +-
.../checks/build.sh => dev-support/bin/test-patch | 7 +-
dev-support/bin/yetus-wrapper | 188 +++++++
dev-support/byteman/README.md | 31 ++
dev-support/byteman/hadooprpc.btm | 44 ++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 11 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 67 +--
.../hdds/scm/client/ContainerOperationClient.java | 10 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 43 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 15 +
.../hadoop/hdds/scm/storage/CommitWatcher.java | 8 +-
.../common/dev-support/findbugsExcludeFile.xml | 5 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 6 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 4 +-
.../hdds/ratis/ContainerCommandRequestMessage.java | 107 ++++
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 17 +-
.../hadoop/hdds/scm/ByteStringConversion.java | 62 +++
.../apache/hadoop/hdds/scm/ByteStringHelper.java | 69 ---
.../hadoop/hdds/scm/container/ContainerInfo.java | 2 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 21 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 15 +-
.../hdds/security/token/BlockTokenVerifier.java | 4 +-
.../security/token/OzoneBlockTokenSelector.java | 8 +-
.../authority/PKIProfiles/DefaultProfile.java | 4 +-
.../hdds/security/x509/keys/HDDSKeyGenerator.java | 6 +-
.../apache/hadoop/hdds/tracing/StringCodec.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 9 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 6 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 20 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 2 +-
.../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 3 +-
.../hadoop/hdds/utils/db/cache/CacheKey.java | 11 +-
.../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 12 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 3 +
.../hadoop/ozone/common/ChecksumByteBuffer.java | 122 +++++
.../ozone/common/PureJavaCrc32ByteBuffer.java | 556 ++++++++++++++++++++
.../ozone/common/PureJavaCrc32CByteBuffer.java | 559 +++++++++++++++++++++
.../helpers/ContainerCommandRequestPBHelper.java | 16 +-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +-
.../apache/hadoop/ozone/lease/LeaseManager.java | 6 +-
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 11 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 19 +-
.../hadoop/ozone/lock/PooledLockFactory.java | 7 +-
.../apache/hadoop/ozone/web/utils/JsonUtils.java | 5 +-
.../common/src/main/resources/ozone-default.xml | 11 +
.../ratis/TestContainerCommandRequestMessage.java | 152 ++++++
.../ozone/common/TestChecksumByteBuffer.java | 102 ++++
.../ozone/container/common/impl/ContainerSet.java | 64 ++-
.../container/common/impl/HddsDispatcher.java | 11 +-
.../RandomContainerDeletionChoosingPolicy.java | 11 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 13 +-
.../common/statemachine/EndpointStateMachine.java | 4 +-
.../CloseContainerCommandHandler.java | 6 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 28 +-
.../server/ratis/ContainerStateMachine.java | 52 +-
.../transport/server/ratis/XceiverServerRatis.java | 5 +-
.../container/common/volume/HddsVolumeChecker.java | 14 +-
.../common/volume/ThrottledAsyncChecker.java | 8 +-
.../container/keyvalue/KeyValueBlockIterator.java | 6 +-
.../container/keyvalue/KeyValueContainerCheck.java | 34 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 117 +++--
.../container/keyvalue/helpers/ChunkUtils.java | 42 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 8 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 6 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 69 ++-
.../keyvalue/interfaces/ChunkManager.java | 2 +-
.../background/BlockDeletingService.java | 6 +-
.../container/ozoneimpl/ContainerController.java | 6 +-
.../container/ozoneimpl/ContainerDataScanner.java | 50 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 4 +-
.../ozoneimpl/ContainerMetadataScanner.java | 19 +-
.../ContainerMetadataScrubberMetrics.java | 5 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 17 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 8 +-
.../container/common/impl/TestContainerSet.java | 18 +-
.../container/keyvalue/TestChunkManagerImpl.java | 69 +--
.../keyvalue/TestKeyValueContainerCheck.java | 69 ++-
.../ozoneimpl/TestContainerScrubberMetrics.java | 25 +-
.../container/ozoneimpl/TestOzoneContainer.java | 23 +-
hadoop-hdds/docs/content/beyond/Containers.md | 2 +-
hadoop-hdds/docs/content/start/FromSource.md | 2 +-
.../hadoop/hdds/server/events/EventQueue.java | 2 +-
hadoop-hdds/pom.xml | 1 -
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 10 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 12 +-
.../scm/command/CommandStatusReportHandler.java | 12 +-
.../container/AbstractContainerReportHandler.java | 6 +-
.../scm/container/ContainerActionsHandler.java | 6 +-
.../hdds/scm/container/ContainerStateManager.java | 4 +-
.../IncrementalContainerReportHandler.java | 6 +-
.../algorithms/SCMContainerPlacementRackAware.java | 6 +-
.../scm/container/states/ContainerAttribute.java | 22 +-
.../scm/container/states/ContainerStateMap.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 10 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 18 +-
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 21 +-
.../hdds/scm/cli/container/ListSubcommand.java | 4 +-
.../client/io/BlockOutputStreamEntryPool.java | 22 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 6 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 97 ++--
.../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 6 +-
.../hadoop/ozone/om/helpers/OMRatisHelper.java | 4 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 139 ++++-
.../security/OzoneBlockTokenSecretManager.java | 2 +-
.../OzoneDelegationTokenSecretManager.java | 6 +-
.../security/OzoneDelegationTokenSelector.java | 8 +-
.../hadoop/ozone/security/OzoneSecretManager.java | 6 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 79 ++-
.../dev-support/checks/_mvn_unit_report.sh | 5 +
hadoop-ozone/dev-support/checks/blockade.sh | 2 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 4 +-
hadoop-ozone/dev-support/checks/integration.sh | 6 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dist/src/main/assemblies/ozone-src.xml | 7 +-
hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env | 2 +-
.../main/compose/ozone-hdfs/docker-compose.yaml | 6 +-
.../dist/src/main/compose/ozone-hdfs/docker-config | 46 --
.../dist/src/main/compose/ozone-mr/common-config | 9 -
.../dist/src/main/compose/ozone-mr/hadoop27/.env | 2 +-
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 2 +-
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop32/.env | 2 +-
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-om-ha/.env | 2 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 10 +-
.../src/main/compose/ozone-om-ha/docker-config | 45 --
.../dist/src/main/compose/ozone-recon/.env | 2 +-
.../main/compose/ozone-recon/docker-compose.yaml | 8 +-
.../src/main/compose/ozone-recon/docker-config | 47 +-
.../dist/src/main/compose/ozone-topology/.env | 2 +-
.../compose/ozone-topology/docker-compose.yaml | 12 +-
.../src/main/compose/ozone-topology/docker-config | 49 --
hadoop-ozone/dist/src/main/compose/ozone/.env | 2 +-
.../src/main/compose/ozone/docker-compose.yaml | 6 +-
.../dist/src/main/compose/ozone/docker-config | 45 --
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
.../dist/src/main/compose/ozoneblockade/.env | 2 +-
.../main/compose/ozoneblockade/docker-compose.yaml | 8 +-
.../src/main/compose/ozoneblockade/docker-config | 45 --
hadoop-ozone/dist/src/main/compose/ozoneperf/.env | 2 +-
.../src/main/compose/ozoneperf/docker-compose.yaml | 10 +-
.../dist/src/main/compose/ozoneperf/docker-config | 13 -
.../dist/src/main/compose/ozones3-haproxy/.env | 2 +-
.../compose/ozones3-haproxy/docker-compose.yaml | 12 +-
.../src/main/compose/ozones3-haproxy/docker-config | 48 --
hadoop-ozone/dist/src/main/compose/ozones3/.env | 2 +-
.../src/main/compose/ozones3/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozones3/docker-config | 48 --
.../dist/src/main/compose/ozonescripts/.env | 2 +-
.../src/main/compose/ozonescripts/docker-config | 7 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../compose/ozonesecure-mr/docker-compose.yaml | 11 +-
.../src/main/compose/ozonesecure-mr/docker-config | 46 --
.../dist/src/main/compose/ozonesecure/.env | 2 +-
.../main/compose/ozonesecure/docker-compose.yaml | 10 +-
.../src/main/compose/ozonesecure/docker-config | 53 --
.../dist/src/main/compose/ozonesecure/test.sh | 2 +
hadoop-ozone/dist/src/main/dockerbin/entrypoint.sh | 2 +-
.../src/main/smoketest/scmcli/pipeline.robot} | 19 +-
.../src/test/blockade/ozone/cluster.py | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 13 +
.../ozone/container/ContainerTestHelper.java | 11 +-
.../container/common/TestBlockDeletingService.java | 24 +-
.../common/impl/TestContainerPersistence.java | 53 +-
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 7 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 4 -
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 19 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 66 ++-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 59 +--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 10 -
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 125 ++++-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 11 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 28 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 8 +-
.../ozone/om/ratis/OzoneManagerRatisClient.java | 53 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 12 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 16 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 13 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 6 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 4 +-
.../om/request/file/OMDirectoryCreateRequest.java | 4 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 4 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 4 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 6 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 4 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 22 +-
.../request/s3/bucket/S3BucketDeleteRequest.java | 13 +-
.../S3InitiateMultipartUploadRequest.java | 4 +-
.../multipart/S3MultipartUploadAbortRequest.java | 4 +-
.../S3MultipartUploadCommitPartRequest.java | 4 +-
.../S3MultipartUploadCompleteRequest.java | 4 +-
.../om/request/s3/security/S3GetSecretRequest.java | 6 +-
.../om/request/volume/OMVolumeCreateRequest.java | 10 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 10 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 4 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 6 +-
.../OzoneManagerHARequestHandlerImpl.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 4 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/web/ozShell/ObjectPrinter.java | 3 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 5 +-
.../web/ozShell/bucket/GetAclBucketHandler.java | 4 +-
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 7 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 5 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 5 +-
.../ozone/web/ozShell/keys/GetAclKeyHandler.java | 4 +-
.../web/ozShell/keys/RemoveAclKeyHandler.java | 7 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 5 +-
.../ozone/web/ozShell/token/GetTokenHandler.java | 2 +-
.../ozone/web/ozShell/token/PrintTokenHandler.java | 2 +-
.../web/ozShell/volume/AddAclVolumeHandler.java | 5 +-
.../web/ozShell/volume/GetAclVolumeHandler.java | 4 +-
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 7 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 5 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 3 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 417 +++++++++++++++
.../ozone/om/request/TestOMRequestUtils.java | 60 ++-
hadoop-ozone/ozonefs-lib-current/pom.xml | 3 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 59 +--
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 +-
hadoop-ozone/pom.xml | 1 -
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 61 +++
.../apache/hadoop/ozone/recon/TestReconUtils.java | 44 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 6 +-
.../apache/hadoop/ozone/s3/AWSV4AuthParser.java | 10 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 5 +-
.../ozone/s3/exception/OS3ExceptionMapper.java | 4 +-
pom.ozone.xml => pom.xml | 0
253 files changed, 5036 insertions(+), 1757 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org