You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by so...@apache.org on 2021/06/07 09:20:34 UTC
[ozone] branch HDDS-3816-ec updated (f904539 -> fbea53d)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a change to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from f904539 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec
add 183f4b4 HDDS-5242. Skip `failing` acceptance suite by default (#2263)
add 18f0fe1 HDDS-4733. Remove leveldb from codebase (#2274)
add 656339e HDDS-5204. Missing type-level Javadoc comments (#2262)
add 90a8d7c HDDS-5276. Use built-in cancel support for duplicates (#2284)
add f3f258a HDDS-5277. Recon shows operational status as "DECOMMISSIONING" for "DECOMMISSIONED" DNs (#2286)
add b6a035a HDDS-5288. Recon UI cannot sort DN by operational state (#2298)
add 3030c9f HDDS-5282. Relocate classes copied from Hadoop (#2293)
add 095d09b HDDS-4483. Datanodes should send last processed CRL sequence ID in heartbeats (#2285)
add 25d66d7 HDDS-5154. Add SCM GRPC server to publish CRL update. (#2216)
add c2ae90b HDDS-5295. testCRLStatusReportPublisher fails to create CRLInfo (#2302)
add 8c1de61 HDDS-5274. Revert "HDDS-5153. Decommissioning a dead node should complete immediately (#2190)" (#2282)
add bb8eb39 HDDS-5286. Document SCM HA classes (#2297)
add 405925f HDDS-5168. Use ReplicationConfig in OmKeyArgs (#2287). Contributed by elek.
add 7bc1a25 HDDS-5292. Introduce the WritableContainerInterface to SCM (#2300)
new fbea53d Merge branch 'master' into EC branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/cancel-ci.yaml | 8 -
.github/workflows/post-commit.yml | 5 +-
.../hdds/scm/storage/TestBlockInputStream.java | 2 +-
.../hdds/scm/storage/TestChunkInputStream.java | 2 +-
hadoop-hdds/common/pom.xml | 5 -
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 8 +-
.../hadoop/hdds/scm/exceptions/SCMException.java | 4 +-
.../scm/ha/RetriableWithNoFailoverException.java | 4 +
.../hadoop/hdds/scm/utils/ClientCommandsUtils.java | 3 +
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 39 ++
.../apache/hadoop/hdds/utils/EntryConsumer.java | 38 --
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +-
.../ozone/common/ChecksumByteBufferFactory.java | 3 +
.../ozone/common/ChecksumByteBufferImpl.java | 5 +-
.../hadoop/ozone/common/utils/BufferUtils.java | 3 +
.../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 3 +
.../common/src/main/resources/ozone-default.xml | 8 +
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 2 +-
.../hdds/fs/TestCachingSpaceUsageSource.java | 2 +-
.../java/org/apache/hadoop/hdds/fs/TestDU.java | 2 +-
.../org/apache/hadoop/hdds/fs/TestDUFactory.java | 2 +-
.../hdds/fs/TestDedicatedDiskSpaceUsage.java | 2 +-
.../fs/TestDedicatedDiskSpaceUsageFactory.java | 2 +-
.../hadoop/hdds/fs/TestSaveSpaceUsageToFile.java | 4 +-
.../hadoop/hdds/fs/TestSpaceUsageFactory.java | 2 +-
.../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 3 +
.../hadoop/hdds/tracing/TestStringCodec.java | 2 +-
.../hadoop/hdds/utils/TestResourceLimitCache.java | 2 +-
.../common/TestChecksumImplsComputeSameValues.java | 4 +
.../apache/hadoop/ozone/lock/TestLockManager.java | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 2 +-
.../common/report/CRLStatusReportPublisher.java | 82 +++
.../common/report/ReportPublisherFactory.java | 2 +
.../common/statemachine/DatanodeStateMachine.java | 21 +-
.../common/statemachine/StateContext.java | 21 +-
.../container/keyvalue/KeyValueContainerCheck.java | 4 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../background/BlockDeletingService.java | 3 +
.../container/replication/ReplicationServer.java | 3 +
.../replication/ReplicationSupervisor.java | 3 +
.../metadata/TestDatanodeCRLStoreImpl.java | 2 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 2 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 4 +-
.../ozone/container/common/SCMTestUtils.java | 2 +-
.../container/common/TestBlockDeletingService.java | 7 +-
.../container/common/TestDatanodeStateMachine.java | 15 +-
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../common/helpers/TestDatanodeVersionFile.java | 2 +-
.../common/impl/TestContainerDataYaml.java | 2 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 2 +-
.../common/impl/TestContainerPersistence.java | 2 +-
.../container/common/impl/TestContainerSet.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 2 +-
.../common/report/TestReportPublisher.java | 43 ++
.../common/report/TestReportPublisherFactory.java | 13 +-
.../common/statemachine/TestStateContext.java | 74 +--
.../common/volume/TestHddsVolumeChecker.java | 2 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 2 +-
.../container/common/volume/TestVolumeSet.java | 4 +-
.../common/volume/TestVolumeSetDiskChecks.java | 2 +-
.../keyvalue/TestKeyValueBlockIterator.java | 2 +-
.../container/keyvalue/TestKeyValueContainer.java | 2 +-
.../keyvalue/TestKeyValueContainerCheck.java | 2 +-
.../TestKeyValueContainerMarkUnhealthy.java | 2 +-
.../container/keyvalue/TestKeyValueHandler.java | 2 +-
.../container/keyvalue/TestTarContainerPacker.java | 2 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 4 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 2 +-
.../keyvalue/impl/TestBlockManagerImpl.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../replication/TestReplicationSupervisor.java | 2 +-
.../replication/TestSimpleContainerDownloader.java | 4 +-
.../container/stream/TestStreamingServer.java | 2 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 3 +
.../java/org/apache/hadoop/hdds/NodeDetails.java | 5 +-
.../scm/update/client/CRLClientUpdateHandler.java | 201 +++++++
.../hadoop/hdds/scm/update/client/CRLStore.java} | 17 +-
.../hdds/scm/update/client/ClientCRLStore.java | 98 ++++
.../scm/update/client/ClientUpdateHandler.java} | 12 +-
.../client/SCMUpdateClientConfiguration.java | 45 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 218 ++++++++
.../scm/update/client/UpdateServiceConfig.java} | 29 +-
.../hdds/scm/update/client}/package-info.java | 4 +-
.../hdds/scm/update/server/CRLClientInfo.java | 56 ++
.../scm/update/server/SCMUpdateClientInfo.java | 67 +++
.../hdds/scm/update/server}/package-info.java | 4 +-
.../x509/certificate/client/CertificateClient.java | 26 +-
.../client/DefaultCertificateClient.java | 65 +++
.../certificate/client/OMCertificateClient.java | 11 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 400 --------------
.../hadoop/hdds/utils/LevelDBStoreIterator.java | 64 ---
.../apache/hadoop/hdds/utils/MetadataStore.java | 233 --------
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 151 ------
.../org/apache/hadoop/hdds/utils/RocksDBStore.java | 397 --------------
.../hadoop/hdds/utils/RocksDBStoreIterator.java | 66 ---
.../hadoop/hdds/utils/db/cache/TableCache.java | 3 +
.../token/TestOzoneBlockTokenIdentifier.java | 2 +-
.../certificate/authority/TestDefaultCAServer.java | 2 +-
.../client/TestCertificateClientInit.java | 2 +-
.../client/TestDefaultCertificateClient.java | 4 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 2 +-
.../hdds/security/x509/keys/TestKeyCodec.java | 2 +-
.../hadoop/hdds/utils/TestMetadataStore.java | 592 ---------------------
.../hadoop/hdds/utils/TestRocksDBStoreMBean.java | 230 --------
.../hdds/utils/db/TestTypedRDBTableStore.java | 2 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 2 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 -
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +
hadoop-hdds/interface-client/pom.xml | 2 +
.../src/main/proto/SCMUpdateProtocol.proto | 98 ++++
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 12 +
.../src/main/proto/ScmServerProtocol.proto | 2 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 89 +---
.../scm/block/DeletedBlockLogStateManager.java | 3 +
.../scm/block/DeletedBlockLogStateManagerImpl.java | 3 +
.../scm/container/balancer/ContainerBalancer.java | 4 +
.../balancer/ContainerBalancerMetrics.java | 3 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 5 +-
.../hdds/scm/ha/MockSCMHADBTransactionBuffer.java | 4 +
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 3 +
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 8 +
.../apache/hadoop/hdds/scm/ha/io/BooleanCodec.java | 3 +
.../org/apache/hadoop/hdds/scm/ha/io/Codec.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/EnumCodec.java | 3 +
.../hdds/scm/ha/io/GeneratedMessageCodec.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/LongCodec.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/StringCodec.java | 3 +
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 3 +
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 3 +
.../hdds/scm/node/NodeDecommissionManager.java | 31 +-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 2 +-
.../scm/pipeline/WritableContainerFactory.java | 58 ++
.../scm/pipeline/WritableContainerProvider.java | 57 ++
.../pipeline/WritableRatisContainerProvider.java | 158 ++++++
.../hadoop/hdds/scm/server/SCMCertStore.java | 3 +
.../hadoop/hdds/scm/server/SCMConfigurator.java | 20 +
.../hdds/scm/server/SCMSecurityProtocolServer.java | 25 +-
.../hdds/scm/server/StorageContainerManager.java | 16 +
.../hadoop/hdds/scm/update/server/SCMCRLStore.java | 51 ++
.../scm/update/server/SCMCRLUpdateHandler.java | 151 ++++++
.../scm/update/server/SCMUpdateClientManager.java | 149 ++++++
.../hdds/scm/update/server/SCMUpdateHandler.java | 53 ++
.../update/server/SCMUpdateServiceGrpcServer.java | 99 ++++
.../scm/update/server/SCMUpdateServiceImpl.java | 119 +++++
.../states => update/server}/package-info.java | 4 +-
.../scm/TestStorageContainerManagerHttpServer.java | 2 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 2 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 +-
.../command/TestCommandStatusReportHandler.java | 2 +-
.../container/TestCloseContainerEventHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 2 +-
.../TestIncrementalContainerReportHandler.java | 2 +-
.../hdds/scm/container/TestReplicationManager.java | 2 +-
.../scm/container/TestSCMContainerManager.java | 2 +-
.../container/balancer/TestContainerBalancer.java | 3 +
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 3 +
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 3 +
.../hdds/scm/ha/TestSequenceIDGenerator.java | 3 +
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 4 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 35 +-
.../hdds/scm/node/TestNodeReportHandler.java | 2 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 2 +-
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 2 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 2 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 2 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 4 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 2 +-
.../TestOneReplicaPipelineSafeModeRule.java | 2 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 2 +-
.../scm/server/TestSCMBlockProtocolServer.java | 2 +-
.../scm/server/TestSCMSecurityProtocolServer.java | 4 +-
.../hdds/scm/update/server/MockCRLStore.java | 138 +++++
.../server/TestSCMUpdateServiceGrpcServer.java | 301 +++++++++++
.../ozone/container/common/TestEndPoint.java | 4 +-
.../{hadoop => ozone}/test/GenericTestUtils.java | 2 +-
.../{hadoop => ozone}/test/LambdaTestUtils.java | 2 +-
.../test/TimedOutTestsListener.java | 2 +-
.../{hadoop => ozone}/test/package-info.java | 2 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 41 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 20 +
.../hadoop/ozone/client/OzoneMultipartUpload.java | 34 +-
.../client/OzoneMultipartUploadPartListParts.java | 34 +-
.../client/io/BlockOutputStreamEntryPool.java | 7 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 30 +-
.../client/io/MultipartCryptoKeyInputStream.java | 3 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 34 +-
.../hadoop/ozone/client/TestOzoneClient.java | 2 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 46 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 50 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 53 +-
.../hadoop/ozone/om/helpers/OmMultipartUpload.java | 32 +-
.../om/helpers/OmMultipartUploadListParts.java | 25 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 71 ++-
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 2 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 10 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 5 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 16 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 1 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 -
.../dist/src/test/shell/compose_testlib.bats | 13 +
.../src/test/shell/{test2 => failing1}/test.sh | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 7 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 6 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 2 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 4 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 2 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 2 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 3 +
.../metrics/TestSCMContainerManagerMetrics.java | 2 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 4 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +-
.../TestRatisPipelineCreateAndDestroy.java | 2 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 2 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 12 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 6 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 8 +-
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 6 +-
.../hadoop/ozone/TestStorageContainerManager.java | 2 +-
.../ozone/client/CertificateClientTestImpl.java | 14 +
.../ozone/client/rpc/Test2WayCommitInRatis.java | 2 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 9 +-
.../rpc/TestCloseContainerHandlingByClient.java | 39 +-
.../rpc/TestContainerReplicationEndToEnd.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 2 +-
.../TestContainerStateMachineFailureOnRead.java | 2 +-
.../rpc/TestContainerStateMachineFailures.java | 2 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 2 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 10 +-
.../client/rpc/TestFailureHandlingByClient.java | 28 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 7 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 13 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 2 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 4 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 2 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 4 +-
.../ozone/client/rpc/TestWatchForCommit.java | 2 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 5 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 8 +-
.../ozone/container/TestContainerReplication.java | 6 +-
.../apache/hadoop/ozone/container/TestHelper.java | 2 +-
.../commandhandler/TestBlockDeletion.java | 13 +-
.../TestCloseContainerByPipeline.java | 23 +-
.../commandhandler/TestCloseContainerHandler.java | 9 +-
.../commandhandler/TestDeleteContainerHandler.java | 8 +-
.../transport/server/ratis/TestCSMMetrics.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 2 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 2 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 2 +-
.../hadoop/ozone/container}/package-info.java | 4 +-
.../container/server/TestContainerServer.java | 2 +-
.../server/TestSecureContainerServer.java | 2 +-
.../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 2 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 +-
.../ozone/freon/TestHadoopDirTreeGenerator.java | 2 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 4 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 2 +-
.../ozone/om/TestContainerReportWithKeys.java | 6 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 35 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 2 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 2 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 4 +
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 2 +-
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 5 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 2 +-
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 4 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 4 +-
.../ozone/om/parser/TestOMRatisLogParser.java | 2 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 4 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 2 +-
.../ozone/recon/TestReconWithOzoneManager.java | 5 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 2 +-
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 2 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 2 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 9 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 2 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 2 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 2 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 4 +-
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 5 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 7 +-
.../ozone/om/codec/TestOmPrefixInfoCodec.java | 2 +-
.../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 5 +-
.../ozone/om/codec/TestS3SecretValueCodec.java | 2 +-
.../ozone/om/codec/TestTransactionInfoCodec.java | 2 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 7 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 81 ++-
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 11 +
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 2 +-
.../om/request/file/OMDirectoryCreateRequest.java | 6 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 2 +-
.../om/request/key/OMAllocateBlockRequest.java | 2 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 2 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 27 +-
.../S3InitiateMultipartUploadRequest.java | 12 +-
.../multipart/S3MultipartUploadAbortRequest.java | 2 +-
.../S3MultipartUploadCommitPartRequest.java | 2 +-
.../S3MultipartUploadCompleteRequest.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 16 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 10 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 6 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 21 +-
.../org/apache/hadoop/ozone/om/TestOMStorage.java | 2 +-
.../ozone/om/TestOzoneManagerHttpServer.java | 2 +-
.../apache/hadoop/ozone/om/TestTrashService.java | 4 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../ozone/om/request/TestOMRequestUtils.java | 8 +-
.../request/bucket/TestOMBucketCreateRequest.java | 2 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 2 +-
.../request/volume/TestOMVolumeCreateRequest.java | 4 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 2 +-
.../s3/multipart/TestS3MultipartResponse.java | 9 +-
.../security/TestOzoneBlockTokenSecretManager.java | 4 +-
.../TestOzoneDelegationTokenSecretManager.java | 2 +-
.../ozone/security/TestOzoneManagerBlockToken.java | 2 +-
.../ozone/security/TestOzoneTokenIdentifier.java | 2 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 7 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 10 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 3 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 1 -
.../ozone/recon/persistence/ContainerHistory.java | 3 +
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 46 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 79 +++
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 20 +
.../scm/ReconStorageContainerManagerFacade.java | 5 +-
.../recon/spi/StorageContainerServiceProvider.java | 6 +
.../impl/StorageContainerServiceProviderImpl.java | 8 +
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 4 +-
.../src/views/datanodes/datanodes.tsx | 2 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 7 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 2 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 2 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 9 +-
...TestReconIncrementalContainerReportHandler.java | 2 +-
.../ozone/recon/scm/TestReconNodeManager.java | 41 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 5 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 5 +-
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 6 +
.../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 6 +
.../apache/hadoop/ozone/s3/endpoint/S3Owner.java | 3 +
.../hadoop/ozone/s3/signature/SignatureInfo.java | 3 +
.../ozone/s3/TestEmptyContentTypeFilter.java | 5 +-
.../ozone/s3/TestVirtualHostStyleFilter.java | 2 +-
.../signature/TestAuthorizationV4HeaderParser.java | 2 +-
.../signature/TestAuthorizationV4QueryParser.java | 5 +-
.../hadoop/ozone/s3/util/TestOzoneS3Util.java | 2 +-
.../ozone/debug/container/ExportSubcommand.java | 3 +
.../ozone/debug/container/InfoSubcommand.java | 3 +
.../ozone/debug/container/ListSubcommand.java | 3 +
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 5 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 3 +
.../freon/containergenerator/BaseGenerator.java | 3 +
.../freon/containergenerator/GeneratorOm.java | 11 +-
.../hadoop/ozone/genesis/BenchMarkCRCBatch.java | 3 +
.../ozone/genesis/BenchMarkCRCStreaming.java | 3 +
.../ozone/genesis/BenchMarkMetadataStoreReads.java | 70 ---
.../genesis/BenchMarkMetadataStoreWrites.java | 62 ---
.../ozone/genesis/BenchMarkOMKeyAllocation.java | 6 +-
.../ozone/genesis/BenchMarkOzoneManager.java | 13 +-
.../ozone/genesis/BenchMarkRocksDbStore.java | 119 -----
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 32 --
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 562 -------------------
.../TestGenerateOzoneRequiredConfigurations.java | 2 +-
pom.xml | 8 +-
385 files changed, 3820 insertions(+), 4037 deletions(-)
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/EntryConsumer.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CRLStatusReportPublisher.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/CRLClientUpdateHandler.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java => framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/CRLStore.java} (75%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/ClientCRLStore.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java => framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/ClientUpdateHandler.java} (73%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServerConfig.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/SCMUpdateClientConfiguration.java (52%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/SCMUpdateServiceGrpcClient.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java => framework/src/main/java/org/apache/hadoop/hdds/scm/update/client/UpdateServiceConfig.java} (56%)
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states => framework/src/main/java/org/apache/hadoop/hdds/scm/update/client}/package-info.java (91%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/CRLClientInfo.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateClientInfo.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states => framework/src/main/java/org/apache/hadoop/hdds/scm/update/server}/package-info.java (91%)
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStore.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStoreIterator.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStore.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataStoreBuilder.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStore.java
delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreIterator.java
delete mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestMetadataStore.java
delete mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRocksDBStoreMBean.java
create mode 100644 hadoop-hdds/interface-client/src/main/proto/SCMUpdateProtocol.proto
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableContainerFactory.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableContainerProvider.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableRatisContainerProvider.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMCRLStore.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMCRLUpdateHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateClientManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateServiceGrpcServer.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/update/server/SCMUpdateServiceImpl.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/{node/states => update/server}/package-info.java (91%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/update/server/MockCRLStore.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/update/server/TestSCMUpdateServiceGrpcServer.java
rename hadoop-hdds/test-utils/src/main/java/org/apache/{hadoop => ozone}/test/GenericTestUtils.java (99%)
rename hadoop-hdds/test-utils/src/main/java/org/apache/{hadoop => ozone}/test/LambdaTestUtils.java (99%)
rename hadoop-hdds/test-utils/src/main/java/org/apache/{hadoop => ozone}/test/TimedOutTestsListener.java (99%)
copy hadoop-hdds/test-utils/src/main/java/org/apache/{hadoop => ozone}/test/package-info.java (96%)
copy hadoop-ozone/dist/src/test/shell/{test2 => failing1}/test.sh (98%)
copy {hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils => hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container}/package-info.java (91%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconDeadNodeHandler.java
delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java
delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java
delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java
delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge branch 'master' into EC branch
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit fbea53d4f57f4f8ba9fcd742a5de047783bb9cba
Merge: f904539 7bc1a25
Author: S O'Donnell <so...@cloudera.com>
AuthorDate: Fri Jun 4 18:35:08 2021 +0100
Merge branch 'master' into EC branch
.github/workflows/cancel-ci.yaml | 8 -
.github/workflows/post-commit.yml | 5 +-
.../hdds/scm/storage/TestBlockInputStream.java | 2 +-
.../hdds/scm/storage/TestChunkInputStream.java | 2 +-
hadoop-hdds/common/pom.xml | 5 -
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 8 +-
.../hadoop/hdds/scm/exceptions/SCMException.java | 4 +-
.../scm/ha/RetriableWithNoFailoverException.java | 4 +
.../hadoop/hdds/scm/utils/ClientCommandsUtils.java | 3 +
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 39 ++
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +-
.../ozone/common/ChecksumByteBufferFactory.java | 3 +
.../ozone/common/ChecksumByteBufferImpl.java | 5 +-
.../hadoop/ozone/common/utils/BufferUtils.java | 3 +
.../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 3 +
.../common/src/main/resources/ozone-default.xml | 8 +
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 2 +-
.../hdds/fs/TestCachingSpaceUsageSource.java | 2 +-
.../java/org/apache/hadoop/hdds/fs/TestDU.java | 2 +-
.../org/apache/hadoop/hdds/fs/TestDUFactory.java | 2 +-
.../hdds/fs/TestDedicatedDiskSpaceUsage.java | 2 +-
.../fs/TestDedicatedDiskSpaceUsageFactory.java | 2 +-
.../hadoop/hdds/fs/TestSaveSpaceUsageToFile.java | 4 +-
.../hadoop/hdds/fs/TestSpaceUsageFactory.java | 2 +-
.../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 3 +
.../hadoop/hdds/tracing/TestStringCodec.java | 2 +-
.../hadoop/hdds/utils/TestResourceLimitCache.java | 2 +-
.../common/TestChecksumImplsComputeSameValues.java | 4 +
.../apache/hadoop/ozone/lock/TestLockManager.java | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 2 +-
.../common/report/CRLStatusReportPublisher.java | 82 +++
.../common/report/ReportPublisherFactory.java | 2 +
.../common/statemachine/DatanodeStateMachine.java | 21 +-
.../common/statemachine/StateContext.java | 21 +-
.../container/keyvalue/KeyValueContainerCheck.java | 4 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../background/BlockDeletingService.java | 3 +
.../container/replication/ReplicationServer.java | 3 +
.../replication/ReplicationSupervisor.java | 3 +
.../metadata/TestDatanodeCRLStoreImpl.java | 2 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 2 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 4 +-
.../ozone/container/common/SCMTestUtils.java | 2 +-
.../container/common/TestBlockDeletingService.java | 7 +-
.../container/common/TestDatanodeStateMachine.java | 15 +-
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../common/helpers/TestDatanodeVersionFile.java | 2 +-
.../common/impl/TestContainerDataYaml.java | 2 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 2 +-
.../common/impl/TestContainerPersistence.java | 2 +-
.../container/common/impl/TestContainerSet.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 2 +-
.../common/report/TestReportPublisher.java | 43 ++
.../common/report/TestReportPublisherFactory.java | 13 +-
.../common/statemachine/TestStateContext.java | 74 +--
.../common/volume/TestHddsVolumeChecker.java | 2 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 2 +-
.../container/common/volume/TestVolumeSet.java | 4 +-
.../common/volume/TestVolumeSetDiskChecks.java | 2 +-
.../keyvalue/TestKeyValueBlockIterator.java | 2 +-
.../container/keyvalue/TestKeyValueContainer.java | 2 +-
.../keyvalue/TestKeyValueContainerCheck.java | 2 +-
.../TestKeyValueContainerMarkUnhealthy.java | 2 +-
.../container/keyvalue/TestKeyValueHandler.java | 2 +-
.../container/keyvalue/TestTarContainerPacker.java | 2 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 4 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 2 +-
.../keyvalue/impl/TestBlockManagerImpl.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../replication/TestReplicationSupervisor.java | 2 +-
.../replication/TestSimpleContainerDownloader.java | 4 +-
.../container/stream/TestStreamingServer.java | 2 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 3 +
.../java/org/apache/hadoop/hdds/NodeDetails.java | 5 +-
.../scm/update/client/CRLClientUpdateHandler.java | 201 +++++++
.../hadoop/hdds/scm/update/client/CRLStore.java} | 24 +-
.../hdds/scm/update/client/ClientCRLStore.java | 98 ++++
.../scm/update/client/ClientUpdateHandler.java} | 23 +-
.../client/SCMUpdateClientConfiguration.java | 53 ++
.../update/client/SCMUpdateServiceGrpcClient.java | 218 ++++++++
.../scm/update/client/UpdateServiceConfig.java} | 42 +-
.../hdds/scm/update/client/package-info.java} | 24 +-
.../hdds/scm/update/server/CRLClientInfo.java | 56 ++
.../scm/update/server/SCMUpdateClientInfo.java | 67 +++
.../hdds/scm/update/server/package-info.java} | 24 +-
.../x509/certificate/client/CertificateClient.java | 26 +-
.../client/DefaultCertificateClient.java | 65 +++
.../certificate/client/OMCertificateClient.java | 11 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 400 --------------
.../hadoop/hdds/utils/LevelDBStoreIterator.java | 64 ---
.../apache/hadoop/hdds/utils/MetadataStore.java | 233 --------
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 151 ------
.../org/apache/hadoop/hdds/utils/RocksDBStore.java | 397 --------------
.../hadoop/hdds/utils/RocksDBStoreIterator.java | 66 ---
.../hadoop/hdds/utils/db/cache/TableCache.java | 3 +
.../token/TestOzoneBlockTokenIdentifier.java | 2 +-
.../certificate/authority/TestDefaultCAServer.java | 2 +-
.../client/TestCertificateClientInit.java | 2 +-
.../client/TestDefaultCertificateClient.java | 4 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 2 +-
.../hdds/security/x509/keys/TestKeyCodec.java | 2 +-
.../hadoop/hdds/utils/TestMetadataStore.java | 592 ---------------------
.../hadoop/hdds/utils/TestRocksDBStoreMBean.java | 230 --------
.../hdds/utils/db/TestTypedRDBTableStore.java | 2 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 2 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 -
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +
hadoop-hdds/interface-client/pom.xml | 2 +
.../src/main/proto/SCMUpdateProtocol.proto | 98 ++++
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 12 +
.../src/main/proto/ScmServerProtocol.proto | 2 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 89 +---
.../scm/block/DeletedBlockLogStateManager.java | 3 +
.../scm/block/DeletedBlockLogStateManagerImpl.java | 3 +
.../scm/container/balancer/ContainerBalancer.java | 4 +
.../balancer/ContainerBalancerMetrics.java | 3 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 5 +-
.../hdds/scm/ha/MockSCMHADBTransactionBuffer.java | 4 +
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 3 +
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 8 +
.../apache/hadoop/hdds/scm/ha/io/BooleanCodec.java | 3 +
.../org/apache/hadoop/hdds/scm/ha/io/Codec.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/EnumCodec.java | 3 +
.../hdds/scm/ha/io/GeneratedMessageCodec.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/LongCodec.java | 3 +
.../apache/hadoop/hdds/scm/ha/io/StringCodec.java | 3 +
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 3 +
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 3 +
.../hdds/scm/node/NodeDecommissionManager.java | 31 +-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 2 +-
.../scm/pipeline/WritableContainerFactory.java | 58 ++
.../scm/pipeline/WritableContainerProvider.java | 57 ++
.../pipeline/WritableRatisContainerProvider.java | 158 ++++++
.../hadoop/hdds/scm/server/SCMCertStore.java | 3 +
.../hadoop/hdds/scm/server/SCMConfigurator.java | 20 +
.../hdds/scm/server/SCMSecurityProtocolServer.java | 25 +-
.../hdds/scm/server/StorageContainerManager.java | 16 +
.../hadoop/hdds/scm/update/server/SCMCRLStore.java | 51 ++
.../scm/update/server/SCMCRLUpdateHandler.java | 151 ++++++
.../scm/update/server/SCMUpdateClientManager.java | 149 ++++++
.../hdds/scm/update/server/SCMUpdateHandler.java | 53 ++
.../update/server/SCMUpdateServiceGrpcServer.java | 99 ++++
.../scm/update/server/SCMUpdateServiceImpl.java | 119 +++++
.../Codec.java => update/server/package-info.java} | 24 +-
.../scm/TestStorageContainerManagerHttpServer.java | 2 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 2 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 +-
.../command/TestCommandStatusReportHandler.java | 2 +-
.../container/TestCloseContainerEventHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 2 +-
.../TestIncrementalContainerReportHandler.java | 2 +-
.../hdds/scm/container/TestReplicationManager.java | 2 +-
.../scm/container/TestSCMContainerManager.java | 2 +-
.../container/balancer/TestContainerBalancer.java | 3 +
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 3 +
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 3 +
.../hdds/scm/ha/TestSequenceIDGenerator.java | 3 +
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 4 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 35 +-
.../hdds/scm/node/TestNodeReportHandler.java | 2 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 2 +-
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 2 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 2 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 2 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 4 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 2 +-
.../TestOneReplicaPipelineSafeModeRule.java | 2 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 2 +-
.../scm/server/TestSCMBlockProtocolServer.java | 2 +-
.../scm/server/TestSCMSecurityProtocolServer.java | 4 +-
.../hdds/scm/update/server/MockCRLStore.java | 138 +++++
.../server/TestSCMUpdateServiceGrpcServer.java | 301 +++++++++++
.../ozone/container/common/TestEndPoint.java | 4 +-
.../{hadoop => ozone}/test/GenericTestUtils.java | 2 +-
.../{hadoop => ozone}/test/LambdaTestUtils.java | 2 +-
.../test/TimedOutTestsListener.java | 2 +-
.../java/org/apache/ozone/test/package-info.java} | 12 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 41 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 20 +
.../hadoop/ozone/client/OzoneMultipartUpload.java | 34 +-
.../client/OzoneMultipartUploadPartListParts.java | 34 +-
.../client/io/BlockOutputStreamEntryPool.java | 7 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 30 +-
.../client/io/MultipartCryptoKeyInputStream.java | 3 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 34 +-
.../hadoop/ozone/client/TestOzoneClient.java | 2 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 46 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 50 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 53 +-
.../hadoop/ozone/om/helpers/OmMultipartUpload.java | 32 +-
.../om/helpers/OmMultipartUploadListParts.java | 25 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 71 ++-
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 2 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 10 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 5 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 16 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 1 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 -
.../dist/src/test/shell/compose_testlib.bats | 13 +
.../{compose_testlib.bats => failing1/test.sh} | 22 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 7 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 6 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 2 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 4 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 2 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 2 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 3 +
.../metrics/TestSCMContainerManagerMetrics.java | 2 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 4 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +-
.../TestRatisPipelineCreateAndDestroy.java | 2 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 2 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 12 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 6 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 8 +-
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 6 +-
.../hadoop/ozone/TestStorageContainerManager.java | 2 +-
.../ozone/client/CertificateClientTestImpl.java | 14 +
.../ozone/client/rpc/Test2WayCommitInRatis.java | 2 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 9 +-
.../rpc/TestCloseContainerHandlingByClient.java | 39 +-
.../rpc/TestContainerReplicationEndToEnd.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 2 +-
.../TestContainerStateMachineFailureOnRead.java | 2 +-
.../rpc/TestContainerStateMachineFailures.java | 2 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 2 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 10 +-
.../client/rpc/TestFailureHandlingByClient.java | 28 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 7 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 13 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 2 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 4 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 2 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 4 +-
.../ozone/client/rpc/TestWatchForCommit.java | 2 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 5 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 8 +-
.../ozone/container/TestContainerReplication.java | 6 +-
.../apache/hadoop/ozone/container/TestHelper.java | 2 +-
.../commandhandler/TestBlockDeletion.java | 13 +-
.../TestCloseContainerByPipeline.java | 23 +-
.../commandhandler/TestCloseContainerHandler.java | 9 +-
.../commandhandler/TestDeleteContainerHandler.java | 8 +-
.../transport/server/ratis/TestCSMMetrics.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 2 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 2 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 2 +-
.../hadoop/ozone/container/package-info.java | 30 +-
.../container/server/TestContainerServer.java | 2 +-
.../server/TestSecureContainerServer.java | 2 +-
.../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 2 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 +-
.../ozone/freon/TestHadoopDirTreeGenerator.java | 2 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 4 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 2 +-
.../ozone/om/TestContainerReportWithKeys.java | 6 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 35 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 2 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 2 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 4 +
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 2 +-
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 5 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 2 +-
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 4 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 4 +-
.../ozone/om/parser/TestOMRatisLogParser.java | 2 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 4 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 2 +-
.../ozone/recon/TestReconWithOzoneManager.java | 5 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 2 +-
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 2 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 2 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 9 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 2 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 2 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 2 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 4 +-
.../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 5 +-
.../om/codec/TestOmMultipartKeyInfoCodec.java | 7 +-
.../ozone/om/codec/TestOmPrefixInfoCodec.java | 2 +-
.../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 5 +-
.../ozone/om/codec/TestS3SecretValueCodec.java | 2 +-
.../ozone/om/codec/TestTransactionInfoCodec.java | 2 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 7 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 81 ++-
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 11 +
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 2 +-
.../om/request/file/OMDirectoryCreateRequest.java | 6 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 2 +-
.../om/request/key/OMAllocateBlockRequest.java | 2 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 2 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 27 +-
.../S3InitiateMultipartUploadRequest.java | 12 +-
.../multipart/S3MultipartUploadAbortRequest.java | 2 +-
.../S3MultipartUploadCommitPartRequest.java | 2 +-
.../S3MultipartUploadCompleteRequest.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 16 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 10 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 6 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 21 +-
.../org/apache/hadoop/ozone/om/TestOMStorage.java | 2 +-
.../ozone/om/TestOzoneManagerHttpServer.java | 2 +-
.../apache/hadoop/ozone/om/TestTrashService.java | 4 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../ozone/om/request/TestOMRequestUtils.java | 8 +-
.../request/bucket/TestOMBucketCreateRequest.java | 2 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 2 +-
.../request/volume/TestOMVolumeCreateRequest.java | 4 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 2 +-
.../s3/multipart/TestS3MultipartResponse.java | 9 +-
.../security/TestOzoneBlockTokenSecretManager.java | 4 +-
.../TestOzoneDelegationTokenSecretManager.java | 2 +-
.../ozone/security/TestOzoneManagerBlockToken.java | 2 +-
.../ozone/security/TestOzoneTokenIdentifier.java | 2 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 7 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 10 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 3 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 1 -
.../ozone/recon/persistence/ContainerHistory.java | 3 +
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 46 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 79 +++
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 20 +
.../scm/ReconStorageContainerManagerFacade.java | 5 +-
.../recon/spi/StorageContainerServiceProvider.java | 6 +
.../impl/StorageContainerServiceProviderImpl.java | 8 +
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 4 +-
.../src/views/datanodes/datanodes.tsx | 2 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 7 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 2 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 2 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 9 +-
...TestReconIncrementalContainerReportHandler.java | 2 +-
.../ozone/recon/scm/TestReconNodeManager.java | 41 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 5 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 5 +-
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 6 +
.../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 6 +
.../apache/hadoop/ozone/s3/endpoint/S3Owner.java | 3 +
.../hadoop/ozone/s3/signature/SignatureInfo.java | 3 +
.../ozone/s3/TestEmptyContentTypeFilter.java | 5 +-
.../ozone/s3/TestVirtualHostStyleFilter.java | 2 +-
.../signature/TestAuthorizationV4HeaderParser.java | 2 +-
.../signature/TestAuthorizationV4QueryParser.java | 5 +-
.../hadoop/ozone/s3/util/TestOzoneS3Util.java | 2 +-
.../ozone/debug/container/ExportSubcommand.java | 3 +
.../ozone/debug/container/InfoSubcommand.java | 3 +
.../ozone/debug/container/ListSubcommand.java | 3 +
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 5 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 3 +
.../freon/containergenerator/BaseGenerator.java | 3 +
.../freon/containergenerator/GeneratorOm.java | 11 +-
.../hadoop/ozone/genesis/BenchMarkCRCBatch.java | 3 +
.../ozone/genesis/BenchMarkCRCStreaming.java | 3 +
.../ozone/genesis/BenchMarkMetadataStoreReads.java | 70 ---
.../genesis/BenchMarkMetadataStoreWrites.java | 62 ---
.../ozone/genesis/BenchMarkOMKeyAllocation.java | 6 +-
.../ozone/genesis/BenchMarkOzoneManager.java | 13 +-
.../ozone/genesis/BenchMarkRocksDbStore.java | 119 -----
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 32 --
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 562 -------------------
.../TestGenerateOzoneRequiredConfigurations.java | 2 +-
pom.xml | 8 +-
384 files changed, 3885 insertions(+), 4089 deletions(-)
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
index e859b13,c4e3ec4..a9642e7
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java
@@@ -27,8 -26,7 +27,8 @@@ import org.apache.hadoop.ozone.OzoneCon
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.keyvalue.ChunkLayoutTestInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
- import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+ import org.apache.ozone.test.GenericTestUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org