You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2019/10/14 15:20:10 UTC
[hadoop-ozone] branch HDDS-1880-Decom updated (c09e0a1 -> 5547d85)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a change to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from c09e0a1 Merge remote-tracking branch 'origin/trunk' into HDDS-1880-Decom
add 5119799 HDDS-2216. Rename HADOOP_RUNNER_VERSION to OZONE_RUNNER_VERSION in compose .env files.
add a6b8b16 HDDS-2222. Add a method to update ByteBuffer in PureJavaCrc32/PureJavaCrc32C. (#1595)
add 6934402 HDDS-2230. Invalid entries in ozonesecure-mr config. (Addendum)
add b6461c3 HDDS-2237. KeyDeletingService throws NPE if it's started too early (#1584)
add c7232bb HDDS-2164 : om.db.checkpoints is getting filling up fast. (#1536)
add e87d6df HDDS-2158. Fixing Json Injection Issue in JsonUtils. (#1486)
add 5583014 HDDS-2250. Generated configs missing from ozone-filesystem-lib jars
add 37bc8b0 HDDS-2257. Fix checkstyle issues in ChecksumByteBuffer (#1603)
add 4d285c6 HDDS-2251. Add an option to customize unit.sh and integration.sh parameters
add e70ea7b HDDS-2169. Avoid buffer copies while submitting client requests in Ratis. Contributed by Tsz-wo Sze(#1517).
add ed47ffe HDDS-2252. Enable gdpr robot test in daily build
add 83a5715 HDDS-2239. Fix TestOzoneFsHAUrls (#1600)
add 719240c HDDS-2238. Container Data Scrubber spams log in empty cluster
add dd2bafa HDDS-2264. Improve output of TestOzoneContainer
add 796c057 HDDS-2259. Container Data Scrubber computes wrong checksum
add a7fdbf2 HDDS-2262. SLEEP_SECONDS: command not found
add e1eb29d HDDS-2245. Use dynamic ports for SCM in TestSecureOzoneCluster Contributed by kevin su.
add 6a580fd HDDS-2260. Avoid evaluation of LOG.trace and LOG.debug statement in the read/write path (HDDS). (#1612)
add 70cf448 HDDS-2244. Use new ReadWrite lock in OzoneManager. (#1589)
add 8ec4ba2 HDDS-2233 - Remove ByteStringHelper and refactor the code to the place where it used (#1596)
add a15782a HDDS-2217. Remove log4j and audit configuration from the docker-config files
add 3d5e915 HDDS-2217. Remove log4j and audit configuration from the docker-config files
add 71cc36f Squashed commit of the following:
add 28e6203 HDDS-2265. integration.sh may report false negative
add 253faed HDDS-2266. Avoid evaluation of LOG.trace and LOG.debug statement in the read/write path. (#1633)
add 9ff77cd HDDS-2269. Provide config for fair/non-fair for OM RW Lock. (#1623)
add f89eda1 HDDS-1984. Fix listBucket API. (#1555)
add 640255a HDDS-1986. Fix listkeys API. (#1588)
add 2862cdd HDDS-2282. scmcli pipeline list command throws NullPointerException. Contributed by Xiaoyu Yao. (#1642)
add 21ef389 HDDS-2213.Reduce key provider loading log level in OzoneFileSystem#getAdditionalTokenIssuers (#1556)
add c8f14a5 HDDS-2287. Import common utility scripts and txt files from Hadoop without history.
add e722e4d HDDS-2290. Rename pom.ozone.xml to pom.xml
new 5547d85 Merge remote-tracking branch 'origin/master' into HDDS-1880-Decom
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
BUILDING.txt | 511 +++++++++++++++++++
.../main/license/src/LICENSE.txt => LICENSE.txt | 53 +-
.../src/main/license/src/NOTICE.txt => NOTICE.txt | 5 +-
README.txt | 7 +
.../hadoop-ozone.sh => dev-support/bin/qbt | 7 +-
.../bin/smart-apply-patch | 7 +-
.../hadoop-ozone.sh => dev-support/bin/test-patch | 7 +-
dev-support/bin/yetus-wrapper | 188 +++++++
.../framework => dev-support/byteman}/README.md | 17 +-
dev-support/byteman/hadooprpc.btm | 44 ++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 11 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 67 +--
.../hdds/scm/client/ContainerOperationClient.java | 10 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 43 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 15 +
.../hadoop/hdds/scm/storage/CommitWatcher.java | 8 +-
.../common/dev-support/findbugsExcludeFile.xml | 5 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 6 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 4 +-
.../hdds/ratis/ContainerCommandRequestMessage.java | 107 ++++
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 17 +-
.../hadoop/hdds/scm/ByteStringConversion.java | 62 +++
.../apache/hadoop/hdds/scm/ByteStringHelper.java | 69 ---
.../hadoop/hdds/scm/container/ContainerInfo.java | 2 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 21 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 15 +-
.../hdds/security/token/BlockTokenVerifier.java | 4 +-
.../security/token/OzoneBlockTokenSelector.java | 8 +-
.../authority/PKIProfiles/DefaultProfile.java | 4 +-
.../hdds/security/x509/keys/HDDSKeyGenerator.java | 6 +-
.../apache/hadoop/hdds/tracing/StringCodec.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 9 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 6 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 20 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 2 +-
.../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 3 +-
.../hadoop/hdds/utils/db/cache/CacheKey.java | 11 +-
.../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 12 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 3 +
.../hadoop/ozone/common/ChecksumByteBuffer.java | 122 +++++
.../ozone/common/PureJavaCrc32ByteBuffer.java | 556 ++++++++++++++++++++
.../ozone/common/PureJavaCrc32CByteBuffer.java | 559 +++++++++++++++++++++
.../helpers/ContainerCommandRequestPBHelper.java | 16 +-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +-
.../apache/hadoop/ozone/lease/LeaseManager.java | 6 +-
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 11 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 19 +-
.../hadoop/ozone/lock/PooledLockFactory.java | 7 +-
.../apache/hadoop/ozone/web/utils/JsonUtils.java | 5 +-
.../common/src/main/resources/ozone-default.xml | 11 +
.../ratis/TestContainerCommandRequestMessage.java | 152 ++++++
.../ozone/common/TestChecksumByteBuffer.java | 102 ++++
.../ozone/container/common/impl/ContainerSet.java | 64 ++-
.../container/common/impl/HddsDispatcher.java | 11 +-
.../RandomContainerDeletionChoosingPolicy.java | 11 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 13 +-
.../common/statemachine/EndpointStateMachine.java | 4 +-
.../CloseContainerCommandHandler.java | 6 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 28 +-
.../server/ratis/ContainerStateMachine.java | 52 +-
.../transport/server/ratis/XceiverServerRatis.java | 5 +-
.../container/common/volume/HddsVolumeChecker.java | 14 +-
.../common/volume/ThrottledAsyncChecker.java | 8 +-
.../container/keyvalue/KeyValueBlockIterator.java | 6 +-
.../container/keyvalue/KeyValueContainerCheck.java | 34 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 117 +++--
.../container/keyvalue/helpers/ChunkUtils.java | 42 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 8 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 6 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 69 ++-
.../keyvalue/interfaces/ChunkManager.java | 2 +-
.../background/BlockDeletingService.java | 6 +-
.../container/ozoneimpl/ContainerController.java | 6 +-
.../container/ozoneimpl/ContainerDataScanner.java | 50 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 4 +-
.../ozoneimpl/ContainerMetadataScanner.java | 19 +-
.../ContainerMetadataScrubberMetrics.java | 5 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 17 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 8 +-
.../container/common/impl/TestContainerSet.java | 18 +-
.../container/keyvalue/TestChunkManagerImpl.java | 69 +--
.../keyvalue/TestKeyValueContainerCheck.java | 69 ++-
.../ozoneimpl/TestContainerScrubberMetrics.java | 25 +-
.../container/ozoneimpl/TestOzoneContainer.java | 23 +-
hadoop-hdds/docs/content/beyond/Containers.md | 2 +-
hadoop-hdds/docs/content/start/FromSource.md | 2 +-
.../hadoop/hdds/server/events/EventQueue.java | 2 +-
hadoop-hdds/pom.xml | 1 -
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 10 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 12 +-
.../scm/command/CommandStatusReportHandler.java | 12 +-
.../container/AbstractContainerReportHandler.java | 6 +-
.../scm/container/ContainerActionsHandler.java | 6 +-
.../hdds/scm/container/ContainerStateManager.java | 4 +-
.../IncrementalContainerReportHandler.java | 6 +-
.../algorithms/SCMContainerPlacementRackAware.java | 6 +-
.../scm/container/states/ContainerAttribute.java | 22 +-
.../scm/container/states/ContainerStateMap.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 10 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 18 +-
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 21 +-
.../hdds/scm/cli/container/ListSubcommand.java | 4 +-
.../client/io/BlockOutputStreamEntryPool.java | 22 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 6 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 97 ++--
.../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 6 +-
.../hadoop/ozone/om/helpers/OMRatisHelper.java | 4 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 139 ++++-
.../security/OzoneBlockTokenSecretManager.java | 2 +-
.../OzoneDelegationTokenSecretManager.java | 6 +-
.../security/OzoneDelegationTokenSelector.java | 8 +-
.../hadoop/ozone/security/OzoneSecretManager.java | 6 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 79 ++-
.../dev-support/checks/_mvn_unit_report.sh | 5 +
hadoop-ozone/dev-support/checks/blockade.sh | 2 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 4 +-
hadoop-ozone/dev-support/checks/integration.sh | 6 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dist/src/main/assemblies/ozone-src.xml | 7 +-
hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env | 2 +-
.../main/compose/ozone-hdfs/docker-compose.yaml | 6 +-
.../dist/src/main/compose/ozone-hdfs/docker-config | 46 --
.../dist/src/main/compose/ozone-mr/common-config | 9 -
.../dist/src/main/compose/ozone-mr/hadoop27/.env | 2 +-
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 2 +-
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop32/.env | 2 +-
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-om-ha/.env | 2 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 10 +-
.../src/main/compose/ozone-om-ha/docker-config | 45 --
.../dist/src/main/compose/ozone-recon/.env | 2 +-
.../main/compose/ozone-recon/docker-compose.yaml | 8 +-
.../src/main/compose/ozone-recon/docker-config | 47 +-
.../dist/src/main/compose/ozone-topology/.env | 2 +-
.../compose/ozone-topology/docker-compose.yaml | 12 +-
.../src/main/compose/ozone-topology/docker-config | 49 --
hadoop-ozone/dist/src/main/compose/ozone/.env | 2 +-
.../src/main/compose/ozone/docker-compose.yaml | 6 +-
.../dist/src/main/compose/ozone/docker-config | 45 --
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
.../dist/src/main/compose/ozoneblockade/.env | 2 +-
.../main/compose/ozoneblockade/docker-compose.yaml | 8 +-
.../src/main/compose/ozoneblockade/docker-config | 45 --
hadoop-ozone/dist/src/main/compose/ozoneperf/.env | 2 +-
.../src/main/compose/ozoneperf/docker-compose.yaml | 10 +-
.../dist/src/main/compose/ozoneperf/docker-config | 13 -
.../dist/src/main/compose/ozones3-haproxy/.env | 2 +-
.../compose/ozones3-haproxy/docker-compose.yaml | 12 +-
.../src/main/compose/ozones3-haproxy/docker-config | 48 --
hadoop-ozone/dist/src/main/compose/ozones3/.env | 2 +-
.../src/main/compose/ozones3/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozones3/docker-config | 48 --
.../dist/src/main/compose/ozonescripts/.env | 2 +-
.../src/main/compose/ozonescripts/docker-config | 7 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../compose/ozonesecure-mr/docker-compose.yaml | 11 +-
.../src/main/compose/ozonesecure-mr/docker-config | 46 --
.../dist/src/main/compose/ozonesecure/.env | 2 +-
.../main/compose/ozonesecure/docker-compose.yaml | 10 +-
.../src/main/compose/ozonesecure/docker-config | 53 --
.../dist/src/main/compose/ozonesecure/test.sh | 2 +
hadoop-ozone/dist/src/main/dockerbin/entrypoint.sh | 2 +-
.../{kinit.robot => scmcli/pipeline.robot} | 14 +-
.../src/test/blockade/ozone/cluster.py | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 13 +
.../ozone/container/ContainerTestHelper.java | 11 +-
.../container/common/TestBlockDeletingService.java | 24 +-
.../common/impl/TestContainerPersistence.java | 53 +-
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 7 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 4 -
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 19 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 66 ++-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 59 +--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 10 -
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 125 ++++-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 11 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 28 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 8 +-
.../ozone/om/ratis/OzoneManagerRatisClient.java | 53 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 12 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 16 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 13 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 6 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 4 +-
.../om/request/file/OMDirectoryCreateRequest.java | 4 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 4 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 4 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 6 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 4 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 22 +-
.../request/s3/bucket/S3BucketDeleteRequest.java | 13 +-
.../S3InitiateMultipartUploadRequest.java | 4 +-
.../multipart/S3MultipartUploadAbortRequest.java | 4 +-
.../S3MultipartUploadCommitPartRequest.java | 4 +-
.../S3MultipartUploadCompleteRequest.java | 4 +-
.../om/request/s3/security/S3GetSecretRequest.java | 6 +-
.../om/request/volume/OMVolumeCreateRequest.java | 10 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 10 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 4 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 6 +-
.../OzoneManagerHARequestHandlerImpl.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 4 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/web/ozShell/ObjectPrinter.java | 3 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 5 +-
.../web/ozShell/bucket/GetAclBucketHandler.java | 4 +-
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 7 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 5 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 5 +-
.../ozone/web/ozShell/keys/GetAclKeyHandler.java | 4 +-
.../web/ozShell/keys/RemoveAclKeyHandler.java | 7 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 5 +-
.../ozone/web/ozShell/token/GetTokenHandler.java | 2 +-
.../ozone/web/ozShell/token/PrintTokenHandler.java | 2 +-
.../web/ozShell/volume/AddAclVolumeHandler.java | 5 +-
.../web/ozShell/volume/GetAclVolumeHandler.java | 4 +-
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 7 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 5 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 3 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 417 +++++++++++++++
.../ozone/om/request/TestOMRequestUtils.java | 60 ++-
hadoop-ozone/ozonefs-lib-current/pom.xml | 3 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 59 +--
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 +-
hadoop-ozone/pom.xml | 1 -
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 61 +++
.../apache/hadoop/ozone/recon/TestReconUtils.java | 44 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 6 +-
.../apache/hadoop/ozone/s3/AWSV4AuthParser.java | 10 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 5 +-
.../ozone/s3/exception/OS3ExceptionMapper.java | 4 +-
pom.ozone.xml => pom.xml | 0
253 files changed, 4758 insertions(+), 1782 deletions(-)
create mode 100644 BUILDING.txt
copy hadoop-ozone/dist/src/main/license/src/LICENSE.txt => LICENSE.txt (82%)
copy hadoop-ozone/dist/src/main/license/src/NOTICE.txt => NOTICE.txt (91%)
create mode 100644 README.txt
copy hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh => dev-support/bin/qbt (83%)
mode change 100644 => 100755
copy hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh => dev-support/bin/smart-apply-patch (84%)
mode change 100644 => 100755
copy hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh => dev-support/bin/test-patch (83%)
mode change 100644 => 100755
create mode 100755 dev-support/bin/yetus-wrapper
copy {hadoop-hdds/framework => dev-support/byteman}/README.md (63%)
create mode 100644 dev-support/byteman/hadooprpc.btm
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/ContainerCommandRequestMessage.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ByteStringConversion.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ByteStringHelper.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBuffer.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32ByteBuffer.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/PureJavaCrc32CByteBuffer.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ratis/TestContainerCommandRequestMessage.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java
copy hadoop-ozone/dist/src/main/smoketest/{kinit.robot => scmcli/pipeline.robot} (74%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
rename pom.ozone.xml => pom.xml (100%)
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 01/01: Merge remote-tracking branch 'origin/master'
into HDDS-1880-Decom
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-1880-Decom
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 5547d850fb0d21e25648ce155b3287795399e068
Merge: c09e0a1 e722e4d
Author: Márton Elek <el...@apache.org>
AuthorDate: Mon Oct 14 16:42:02 2019 +0200
Merge remote-tracking branch 'origin/master' into HDDS-1880-Decom
BUILDING.txt | 511 +++++++++++++++++++
LICENSE.txt | 258 ++++++++++
NOTICE.txt | 34 ++
README.txt | 7 +
.../checks/build.sh => dev-support/bin/qbt | 7 +-
.../build.sh => dev-support/bin/smart-apply-patch | 7 +-
.../checks/build.sh => dev-support/bin/test-patch | 7 +-
dev-support/bin/yetus-wrapper | 188 +++++++
dev-support/byteman/README.md | 31 ++
dev-support/byteman/hadooprpc.btm | 44 ++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 11 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 67 +--
.../hdds/scm/client/ContainerOperationClient.java | 10 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 43 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 15 +
.../hadoop/hdds/scm/storage/CommitWatcher.java | 8 +-
.../common/dev-support/findbugsExcludeFile.xml | 5 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 6 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 4 +-
.../hdds/ratis/ContainerCommandRequestMessage.java | 107 ++++
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 17 +-
.../hadoop/hdds/scm/ByteStringConversion.java | 62 +++
.../apache/hadoop/hdds/scm/ByteStringHelper.java | 69 ---
.../hadoop/hdds/scm/container/ContainerInfo.java | 2 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 21 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 15 +-
.../hdds/security/token/BlockTokenVerifier.java | 4 +-
.../security/token/OzoneBlockTokenSelector.java | 8 +-
.../authority/PKIProfiles/DefaultProfile.java | 4 +-
.../hdds/security/x509/keys/HDDSKeyGenerator.java | 6 +-
.../apache/hadoop/hdds/tracing/StringCodec.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 9 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 6 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 20 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 2 +-
.../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 3 +-
.../hadoop/hdds/utils/db/cache/CacheKey.java | 11 +-
.../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 12 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 3 +
.../hadoop/ozone/common/ChecksumByteBuffer.java | 122 +++++
.../ozone/common/PureJavaCrc32ByteBuffer.java | 556 ++++++++++++++++++++
.../ozone/common/PureJavaCrc32CByteBuffer.java | 559 +++++++++++++++++++++
.../helpers/ContainerCommandRequestPBHelper.java | 16 +-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +-
.../apache/hadoop/ozone/lease/LeaseManager.java | 6 +-
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 11 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 19 +-
.../hadoop/ozone/lock/PooledLockFactory.java | 7 +-
.../apache/hadoop/ozone/web/utils/JsonUtils.java | 5 +-
.../common/src/main/resources/ozone-default.xml | 11 +
.../ratis/TestContainerCommandRequestMessage.java | 152 ++++++
.../ozone/common/TestChecksumByteBuffer.java | 102 ++++
.../ozone/container/common/impl/ContainerSet.java | 64 ++-
.../container/common/impl/HddsDispatcher.java | 11 +-
.../RandomContainerDeletionChoosingPolicy.java | 11 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 13 +-
.../common/statemachine/EndpointStateMachine.java | 4 +-
.../CloseContainerCommandHandler.java | 6 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 28 +-
.../server/ratis/ContainerStateMachine.java | 52 +-
.../transport/server/ratis/XceiverServerRatis.java | 5 +-
.../container/common/volume/HddsVolumeChecker.java | 14 +-
.../common/volume/ThrottledAsyncChecker.java | 8 +-
.../container/keyvalue/KeyValueBlockIterator.java | 6 +-
.../container/keyvalue/KeyValueContainerCheck.java | 34 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 117 +++--
.../container/keyvalue/helpers/ChunkUtils.java | 42 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 8 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 6 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 69 ++-
.../keyvalue/interfaces/ChunkManager.java | 2 +-
.../background/BlockDeletingService.java | 6 +-
.../container/ozoneimpl/ContainerController.java | 6 +-
.../container/ozoneimpl/ContainerDataScanner.java | 50 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 4 +-
.../ozoneimpl/ContainerMetadataScanner.java | 19 +-
.../ContainerMetadataScrubberMetrics.java | 5 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 17 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 8 +-
.../container/common/impl/TestContainerSet.java | 18 +-
.../container/keyvalue/TestChunkManagerImpl.java | 69 +--
.../keyvalue/TestKeyValueContainerCheck.java | 69 ++-
.../ozoneimpl/TestContainerScrubberMetrics.java | 25 +-
.../container/ozoneimpl/TestOzoneContainer.java | 23 +-
hadoop-hdds/docs/content/beyond/Containers.md | 2 +-
hadoop-hdds/docs/content/start/FromSource.md | 2 +-
.../hadoop/hdds/server/events/EventQueue.java | 2 +-
hadoop-hdds/pom.xml | 1 -
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 10 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 12 +-
.../scm/command/CommandStatusReportHandler.java | 12 +-
.../container/AbstractContainerReportHandler.java | 6 +-
.../scm/container/ContainerActionsHandler.java | 6 +-
.../hdds/scm/container/ContainerStateManager.java | 4 +-
.../IncrementalContainerReportHandler.java | 6 +-
.../algorithms/SCMContainerPlacementRackAware.java | 6 +-
.../scm/container/states/ContainerAttribute.java | 22 +-
.../scm/container/states/ContainerStateMap.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 10 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 18 +-
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 21 +-
.../hdds/scm/cli/container/ListSubcommand.java | 4 +-
.../client/io/BlockOutputStreamEntryPool.java | 22 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 6 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 97 ++--
.../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 6 +-
.../hadoop/ozone/om/helpers/OMRatisHelper.java | 4 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 139 ++++-
.../security/OzoneBlockTokenSecretManager.java | 2 +-
.../OzoneDelegationTokenSecretManager.java | 6 +-
.../security/OzoneDelegationTokenSelector.java | 8 +-
.../hadoop/ozone/security/OzoneSecretManager.java | 6 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 79 ++-
.../dev-support/checks/_mvn_unit_report.sh | 5 +
hadoop-ozone/dev-support/checks/blockade.sh | 2 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 4 +-
hadoop-ozone/dev-support/checks/integration.sh | 6 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dist/src/main/assemblies/ozone-src.xml | 7 +-
hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env | 2 +-
.../main/compose/ozone-hdfs/docker-compose.yaml | 6 +-
.../dist/src/main/compose/ozone-hdfs/docker-config | 46 --
.../dist/src/main/compose/ozone-mr/common-config | 9 -
.../dist/src/main/compose/ozone-mr/hadoop27/.env | 2 +-
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 2 +-
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop32/.env | 2 +-
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-om-ha/.env | 2 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 10 +-
.../src/main/compose/ozone-om-ha/docker-config | 45 --
.../dist/src/main/compose/ozone-recon/.env | 2 +-
.../main/compose/ozone-recon/docker-compose.yaml | 8 +-
.../src/main/compose/ozone-recon/docker-config | 47 +-
.../dist/src/main/compose/ozone-topology/.env | 2 +-
.../compose/ozone-topology/docker-compose.yaml | 12 +-
.../src/main/compose/ozone-topology/docker-config | 49 --
hadoop-ozone/dist/src/main/compose/ozone/.env | 2 +-
.../src/main/compose/ozone/docker-compose.yaml | 6 +-
.../dist/src/main/compose/ozone/docker-config | 45 --
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
.../dist/src/main/compose/ozoneblockade/.env | 2 +-
.../main/compose/ozoneblockade/docker-compose.yaml | 8 +-
.../src/main/compose/ozoneblockade/docker-config | 45 --
hadoop-ozone/dist/src/main/compose/ozoneperf/.env | 2 +-
.../src/main/compose/ozoneperf/docker-compose.yaml | 10 +-
.../dist/src/main/compose/ozoneperf/docker-config | 13 -
.../dist/src/main/compose/ozones3-haproxy/.env | 2 +-
.../compose/ozones3-haproxy/docker-compose.yaml | 12 +-
.../src/main/compose/ozones3-haproxy/docker-config | 48 --
hadoop-ozone/dist/src/main/compose/ozones3/.env | 2 +-
.../src/main/compose/ozones3/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozones3/docker-config | 48 --
.../dist/src/main/compose/ozonescripts/.env | 2 +-
.../src/main/compose/ozonescripts/docker-config | 7 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../compose/ozonesecure-mr/docker-compose.yaml | 11 +-
.../src/main/compose/ozonesecure-mr/docker-config | 46 --
.../dist/src/main/compose/ozonesecure/.env | 2 +-
.../main/compose/ozonesecure/docker-compose.yaml | 10 +-
.../src/main/compose/ozonesecure/docker-config | 53 --
.../dist/src/main/compose/ozonesecure/test.sh | 2 +
hadoop-ozone/dist/src/main/dockerbin/entrypoint.sh | 2 +-
.../src/main/smoketest/scmcli/pipeline.robot} | 19 +-
.../src/test/blockade/ozone/cluster.py | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 13 +
.../ozone/container/ContainerTestHelper.java | 11 +-
.../container/common/TestBlockDeletingService.java | 24 +-
.../common/impl/TestContainerPersistence.java | 53 +-
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 7 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 4 -
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 19 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 66 ++-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 59 +--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 10 -
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 125 ++++-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 11 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 28 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 8 +-
.../ozone/om/ratis/OzoneManagerRatisClient.java | 53 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 12 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 16 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 13 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 6 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 4 +-
.../om/request/file/OMDirectoryCreateRequest.java | 4 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 4 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 4 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 6 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 4 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 22 +-
.../request/s3/bucket/S3BucketDeleteRequest.java | 13 +-
.../S3InitiateMultipartUploadRequest.java | 4 +-
.../multipart/S3MultipartUploadAbortRequest.java | 4 +-
.../S3MultipartUploadCommitPartRequest.java | 4 +-
.../S3MultipartUploadCompleteRequest.java | 4 +-
.../om/request/s3/security/S3GetSecretRequest.java | 6 +-
.../om/request/volume/OMVolumeCreateRequest.java | 10 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 10 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 4 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 6 +-
.../OzoneManagerHARequestHandlerImpl.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 4 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/web/ozShell/ObjectPrinter.java | 3 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 5 +-
.../web/ozShell/bucket/GetAclBucketHandler.java | 4 +-
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 7 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 5 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 5 +-
.../ozone/web/ozShell/keys/GetAclKeyHandler.java | 4 +-
.../web/ozShell/keys/RemoveAclKeyHandler.java | 7 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 5 +-
.../ozone/web/ozShell/token/GetTokenHandler.java | 2 +-
.../ozone/web/ozShell/token/PrintTokenHandler.java | 2 +-
.../web/ozShell/volume/AddAclVolumeHandler.java | 5 +-
.../web/ozShell/volume/GetAclVolumeHandler.java | 4 +-
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 7 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 5 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 3 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 417 +++++++++++++++
.../ozone/om/request/TestOMRequestUtils.java | 60 ++-
hadoop-ozone/ozonefs-lib-current/pom.xml | 3 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 59 +--
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 +-
hadoop-ozone/pom.xml | 1 -
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 61 +++
.../apache/hadoop/ozone/recon/TestReconUtils.java | 44 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 6 +-
.../apache/hadoop/ozone/s3/AWSV4AuthParser.java | 10 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 5 +-
.../ozone/s3/exception/OS3ExceptionMapper.java | 4 +-
pom.ozone.xml => pom.xml | 0
253 files changed, 5036 insertions(+), 1757 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org