You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2022/03/03 13:44:38 UTC
[ozone] branch HDDS-6030 updated (c48b559 -> 70a9b9c)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a change to branch HDDS-6030
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from c48b559 Merge remote-tracking branch 'origin/master' into HDDS-6030
add 5d496f2 HDDS-6201. Fix NPE for DataScanner with scanned container deleted by others. (#3005)
add 9186e6b HDDS-5529. For any IOexception from @Replicated method we should throw it (#2788)
add 1804e22 HDDS-6181. Change SCMHAInvocationHandler#invokeRatis() logging to TRACE (#2992)
add e47b6f0 HDDS-6206. Application errors must not flood system log (#3001)
add f757d99 HDDS-6245. Add BucketLayout logging to Audit Logs (#3040)
add aafc21a HDDS-6238 Reduce memory requirements for list keys. (#3032)
add 7cee0ea HDDS-2919. Intermittent failure in TestRDBStore (#3028)
add 5a35cab HDDS-6253. Unnecessary duplicate smoketest after defaulting to FSO (#3036)
add 5b1dbeb HDDS-6204. Cleanup handling malformed authorization header (#2999)
add b32e135 HDDS-6169. Selective checks: skip junit tests on ozone-runner image update (#2974)
add 14fa87c HDDS-6270. Use a dedicated file instead of /etc/passwd for xcompat acceptance test (#3050)
add 5c3aa01 HDDS-6273. Amend doc SecuringTDE.md (#3047)
add c5fb612 HDDS-6140. Selective checks: skip unit check for integration-test changes (#2948)
add 11f4c9b HDDS-6215. Recon get limited delta updates from OM (#3009)
add 40d0a40 HDDS-6226. Run tests for selective CI checks in CI (#3019)
add 86a771d HDDS-6247. Avoid logging stack trace for user input problems (#3039)
add de42c61 HDDS-6208. New checkstyle: WhitespaceAround (#3003)
add ad6d3bc HDDS-6289. Upgrade acceptance test log flooded with parse error (#3063)
add 5f8e637 HDDS-6291. Do not let slf4j-log4j12 leak into Ozone FS shaded jar (#3064)
add 59cae07 HDDS-6285. ozonesecure-mr intermittently failing with timeout downloading packages (#3057)
add a5accd2 HDDS-6278 Improve memory profile for listStatus API call. (#3053)
add 594c4a9 HDDS-6257. Wrong stack trace for S3 errors (#3066)
add f31d589 HDDS-6088. Implement O3FS/OFS getFileChecksum() using file checksum helpers. (#2935)
add 7c990ab HDDS-6088. Implement O3FS/OFS getFileChecksum() using file checksum helpers - addendum: fix checkstyle
add f7e1fa0 HDDS-4126. Freon key generator should support >2GB files. (#3054)
add 9ce981c HDDS-6293. Allow using custom ozone-runner image (#3072)
add fa2ae7b HDDS-6309. Update ozone-runner version to 20220212-1 (#3079)
add 3ed0b6b HDDS-6304. Add enforcer to make sure ozone.version equals project.version (#3075)
add 49221c0 HDDS-6190. Cleanup unnecessary datanode id path checks. (#2993)
add e7f5b4e HDDS-6310. Update json-smart to 2.4.7. (#3080)
add 917ce68 HDDS-6186. Selective checks: skip integration check for unit test changes (#3061)
add 41651e8 HDDS-6313. Remove replicas in ContainerStateMap when a container is deleted (#3086)
add c04c01d HDDS-5913. Avoid integer overflow when setting dfs.container.ratis.lo… (#2785)
add e0d4007 HDDS-6322. Fix Recon getting inccorrect sequenceNumber from OM (#3090)
add 000902d HDDS-6325. Fix interface ClientProtocol methods typo setThreadLocalS3Auth and clearThreadLocalS3Auth (#3093)
add f0f344d HDDS-6311. Fix number of keys displayed in Recon Overview. (#3081)
add 87ff329 HDDS-6305: Add metrics - number of FSO bucket creates (#3077)
add 8c5fefe HDDS-6330. Remove unnecessary duplicate semicolons (#3097)
add ff0209d HDDS-6331. Remove toString in debug log parameters within SCMCommonPlacementPolicy (#3098)
add 2c561d0 HDDS-6283. Change ContainerStateMachine ContainerOpExecutor name (#3055)
add eb0f56b HDDS-6324. Do not trigger CI by reopening PR (#3092)
add c87b059 HDDS-6284. Add BlockDeletingService worker size config (#3056)
add 51f47fc HDDS-6314. ConcurrentModificationException getting SCMContainerMetrics (#3101)
add 53fa44d HDDS-6290. operational-state and node-state options in datanode list CLI not working correctly (#3105)
add 91cf777 HDDS-6262. ozone insight log stops working after OM DBUpdates message (#3044)
add 07fa775 HDDS-6234. Repair containers affected by incorrect used bytes and block count. (#3042)
add 788ff0f HDDS-6212. SCM Container DB bootstrap on Recon startup for secure cluster (#3027)
add ad33ac6 HDDS-6292. Ensure immutable ContainerReplica set is returned from ContainerStateManagerImpl (#3071)
add 8ad0796 HDDS-6329. New checkstyle: AvoidStarImport (#3096)
add 7f6e6fb HDDS-6350. Rename TestStandardOutputUtil to observe naming conventions (#3114)
add 4b935c5 HDDS-6328. Add metrics - number of FSO bucket deletes (#3107)
add e848ab7 HDDS-6327. Upgrade acceptance test doesn't collect logs when the test fails (#3106)
add 68c5ac5 HDDS-6351. [Docs] Update prefix-based FSO documentation. (#3115)
add 239d889 HDDS-5267. Full Container Report can remove replicas added by an Incremental Report (#2963)
add f0ee98a HDDS-6363. Update checkstyle version to 9.3 (#3123)
add 1c7ff7e HDDS-6221. Update versions covered by cross-compatibility test (#3017)
add a195100 HDDS-5359. Incorrect BLOCKCOUNT and BYTESUSED in container DB. (#3034)
add 5f4c31a HDDS-6349. IncompleteReadError on get MPU key from TDE bucket (#3116)
add a53a3a3 HDDS-6235. Empty KeyValueContainers are replicated without chunks directory. (#3052)
add 7310491 HDDS-5194. Provide kustomize descriptors for Ozone kubernetes examples (#2221)
add 93631a1 HDDS-6334. Remove ContainerID to Proto to ContainerID conversion in ContainerStateManagerImpl (#3110)
add 17d86f1 HDDS-6361. Modify docs build flow to replace image tags with shortcodes. (#3122)
add da93105 HDDS-6369. PARTIAL_RENAME does not update OM DB (#3126)
add fbce851 HDDS-6374. Fix incorrect queueTime metrics of ReplicationTask (#3135)
add ebbe9b5 HDDS-6337. [FSO] Disable recursive access check flag for directories with no children. (#3134)
add 898ccbb HDDS-6376. Docs: Fix classpath for ofs and o3fs (#3138)
add 53d25b9 HDDS-6307. Improve processing and memory efficiency of full container reports (#3085)
add 6e117f6 HDDS-6395. Intermittent failure in TestReconScmHASnapshot.testScmHASnapshot (#3149)
add 2ed5994 HDDS-6268. Include audit log in acceptance test bundle (#3045)
add 5f029d0 HDDS-6317: Export ReconTaskStatus as Prometheus metrics (#3088)
add ba31f8c HDDS-6333. Add a metric to record sequence number lag between Recon and OM (#3100)
add e9722b7 HDDS-6401. Fix flaky TestFilePerBlockStrategy.testWriteAndReadChunkMultipleTimes (#3152)
add d0cde3a HDDS-5712. make it configurable to trigger refresh datanode usage info before start a new balance iteration (#2944)
add b83c1f9 HDDS-6244. ContainerBalancer metrics don't show updated values in JMX (#3049)
new 70a9b9c Merge remote-tracking branch 'origin/master' into HDDS-6030
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 8 +-
.gitignore | 3 +-
dev-support/ci/selective_ci_checks.bats | 70 +++-
dev-support/ci/selective_ci_checks.sh | 76 +++-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 38 ++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 8 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 2 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 2 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 2 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +-
.../java/org/apache/hadoop/hdds/StringUtils.java | 2 +-
.../hadoop/hdds/annotation/InterfaceAudience.java | 6 +-
.../org/apache/hadoop/hdds/client/OzoneQuota.java | 2 +-
.../org/apache/hadoop/hdds/client/QuotaList.java | 11 +-
.../hadoop/hdds/client/ReplicationFactor.java | 2 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 2 +-
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 2 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 4 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 2 +-
.../RequestTypeDependentRetryPolicyCreator.java | 6 +-
.../org/apache/hadoop/hdds/recon/ReconConfig.java | 2 +-
.../hadoop/hdds/scm/ByteStringConversion.java | 2 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 2 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 21 +-
.../scm/container/ReplicationManagerReport.java | 2 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 9 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 16 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 28 +-
.../org/apache/hadoop/hdds/scm/net/NodeSchema.java | 4 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 8 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 8 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 8 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 2 +-
.../hadoop/hdds/utils/ResourceSemaphore.java | 10 +-
.../org/apache/hadoop/hdds/utils/UniqueId.java | 2 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 11 +-
.../hadoop/ozone/audit/AuditEventStatus.java | 2 +-
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 4 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 2 +-
.../org/apache/hadoop/ozone/audit/AuditMarker.java | 4 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 10 +-
.../hadoop/ozone/common/ChecksumByteBuffer.java | 2 +-
.../ozone/common/IncrementalChunkBuffer.java | 13 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 8 +-
.../ozone/common/ha/ratis/RatisSnapshotInfo.java | 2 +-
.../ozone/container/common/helpers/BlockData.java | 4 +-
.../helpers/ContainerCommandRequestPBHelper.java | 10 +-
.../java/org/apache/hadoop/ozone/lease/Lease.java | 8 +-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 26 ++
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 4 +-
.../hadoop/hdds/client/TestReplicationConfig.java | 2 +-
.../java/org/apache/hadoop/hdds/fs/TestDU.java | 2 +-
.../ratis/TestContainerCommandRequestMessage.java | 6 +-
.../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 4 +-
.../hdds/scm/net/TestNetworkTopologyImpl.java | 20 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 2 +-
.../hadoop/hdds/utils/MockGatheringChannel.java | 2 +-
.../hadoop/hdds/utils/TestResourceSemaphore.java | 6 +-
.../org/apache/hadoop/ozone/audit/DummyEntity.java | 2 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 4 +-
.../apache/hadoop/ozone/common/TestChecksum.java | 2 +-
.../hadoop/ozone/common/TestChunkBuffer.java | 20 +-
.../hadoop/ozone/common/TestStateMachine.java | 4 +-
.../ozone/container/ContainerTestHelper.java | 70 +---
.../hadoop/ozone/lease/TestLeaseManager.java | 2 +-
.../TestDefaultUpgradeFinalizationExecutor.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 27 +-
.../container/common/helpers/ContainerMetrics.java | 6 +-
.../container/common/helpers/ContainerUtils.java | 2 +-
.../ozone/container/common/impl/ContainerData.java | 46 +-
.../ozone/container/common/impl/ContainerSet.java | 16 +-
.../container/common/impl/HddsDispatcher.java | 15 +-
.../common/impl/OpenContainerBlockMap.java | 2 +-
.../common/interfaces/ContainerInspector.java | 72 ++++
.../ozone/container/common/interfaces/Handler.java | 12 +-
.../common/report/ContainerReportPublisher.java | 5 +-
.../common/report/IncrementalReportSender.java} | 16 +-
.../common/statemachine/DatanodeConfiguration.java | 2 +-
.../common/statemachine/DatanodeStateMachine.java | 14 +-
.../common/statemachine/StateContext.java | 143 ++++---
.../commandhandler/CommandDispatcher.java | 2 +-
.../SetNodeOperationalStateCommandHandler.java | 11 -
.../common/states/datanode/InitDatanodeState.java | 9 +-
.../states/datanode/RunningDatanodeState.java | 8 +-
.../states/endpoint/HeartbeatEndpointTask.java | 23 +-
.../states/endpoint/VersionEndpointTask.java | 2 +-
.../common/transport/server/ratis/CSMMetrics.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 11 +-
.../transport/server/ratis/XceiverServerRatis.java | 9 +-
.../common/utils/ContainerInspectorUtil.java | 87 ++++
.../container/common/utils/HddsVolumeUtil.java | 2 +-
.../container/common/volume/AbstractFuture.java | 13 +-
.../container/common/volume/MutableVolumeSet.java | 2 +-
.../container/common/volume/StorageVolume.java | 2 +-
.../ozone/container/common/volume/VolumeUsage.java | 2 +-
.../container/keyvalue/KeyValueContainer.java | 82 +++-
.../container/keyvalue/KeyValueContainerCheck.java | 6 +-
.../container/keyvalue/KeyValueContainerData.java | 12 +-
.../KeyValueContainerMetadataInspector.java | 463 +++++++++++++++++++++
.../ozone/container/keyvalue/KeyValueHandler.java | 101 ++---
.../container/keyvalue/TarContainerPacker.java | 55 ++-
.../container/keyvalue/helpers/ChunkUtils.java | 4 +
.../helpers/KeyValueContainerLocationUtil.java | 2 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 23 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 110 +++--
.../keyvalue/impl/ChunkManagerDispatcher.java | 18 +-
.../background/BlockDeletingService.java | 112 +++--
.../container/metadata/AbstractDatanodeStore.java | 8 +-
.../container/ozoneimpl/ContainerController.java | 14 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 6 +-
.../ozoneimpl/ContainerMetadataScanner.java | 2 +-
.../ContainerMetadataScrubberMetrics.java | 2 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 6 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 40 +-
.../replication/GrpcReplicationClient.java | 2 +-
.../container/replication/MeasuredReplicator.java | 3 +-
.../container/stream/DirstreamClientHandler.java | 2 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 2 +-
.../upgrade/VersionedDatanodeFeatures.java | 2 +-
.../commands/RefreshVolumeUsageCommand.java | 57 +++
.../ozone/protocol/commands/ReregisterCommand.java | 2 +-
.../hadoop/ozone/container/common/ScmTestMock.java | 42 +-
.../container/common/TestBlockDeletingService.java | 13 +-
.../ozone/container/common/TestContainerCache.java | 2 +-
.../container/common/TestDatanodeStateMachine.java | 5 -
.../common/TestKeyValueContainerData.java | 6 +-
.../TestSchemaOneBackwardsCompatibility.java | 38 +-
.../container/common/helpers/TestBlockData.java | 6 +-
.../common/helpers/TestDatanodeVersionFile.java | 14 +-
.../common/impl/TestContainerDataYaml.java | 2 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 2 +-
.../common/impl/TestContainerPersistence.java | 47 ---
.../container/common/impl/TestContainerSet.java | 12 +-
.../container/common/impl/TestHddsDispatcher.java | 7 +-
.../container/common/interfaces/TestHandler.java | 2 +-
.../common/report/TestReportPublisher.java | 2 +-
.../common/statemachine/TestStateContext.java | 71 +++-
.../TestCloseContainerCommandHandler.java | 4 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 2 +-
.../common/volume/TestStorageVolumeChecker.java | 12 +-
.../container/common/volume/TestVolumeSet.java | 2 +-
.../keyvalue/TestKeyValueBlockIterator.java | 24 +-
.../container/keyvalue/TestKeyValueContainer.java | 69 ++-
.../keyvalue/TestKeyValueContainerCheck.java | 160 +------
...a => TestKeyValueContainerIntegrityChecks.java} | 160 ++-----
.../TestKeyValueContainerMetadataInspector.java | 360 ++++++++++++++++
.../container/keyvalue/TestKeyValueHandler.java | 23 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 14 -
.../keyvalue/impl/CommonChunkManagerTestCases.java | 5 +-
.../keyvalue/impl/TestBlockManagerImpl.java | 57 +--
.../container/ozoneimpl/TestContainerReader.java | 10 +-
.../container/ozoneimpl/TestOzoneContainer.java | 6 +-
.../replication/TestGrpcOutputStream.java | 8 +-
.../replication/TestMeasuredReplicator.java | 15 +
.../replication/TestReplicationSupervisor.java | 2 +-
.../testutils/BlockDeletingServiceTestImpl.java | 2 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 6 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 2 +
hadoop-hdds/docs/content/concept/Containers.md | 3 +-
hadoop-hdds/docs/content/concept/Datanodes.md | 4 +-
hadoop-hdds/docs/content/concept/OzoneManager.md | 6 +-
hadoop-hdds/docs/content/concept/Recon.md | 9 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 4 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 68 ++-
hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Ofs.md | 2 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 48 ++-
.../docs/content/security/SecuringTDE.zh.md | 4 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 +++
.../themes/ozonedoc/layouts/shortcodes/image.html | 2 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 12 +
.../SCMSecurityProtocolClientSideTranslatorPB.java | 39 +-
...inerLocationProtocolClientSideTranslatorPB.java | 4 +-
.../scm/update/client/CRLClientUpdateHandler.java | 2 +-
.../hdds/scm/update/client/ClientCRLStore.java | 4 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 2 +-
.../scm/update/client/UpdateServiceConfig.java | 5 +-
.../x509/certificate/authority/BaseApprover.java | 2 +-
.../certificate/authority/DefaultCAServer.java | 6 +-
.../authority/PKIProfiles/DefaultCAProfile.java | 2 +-
.../authority/PKIProfiles/DefaultProfile.java | 32 +-
...ateClient.java => CommonCertificateClient.java} | 116 +++---
.../client/DefaultCertificateClient.java | 32 +-
.../certificate/client/OMCertificateClient.java | 84 +---
.../certificate/client/ReconCertificateClient.java | 38 +-
.../certificates/utils/CertificateSignRequest.java | 10 +-
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 2 +-
.../hdds/security/x509/crl/CRLInfoCodec.java | 2 +-
...va => FixedThreadPoolWithAffinityExecutor.java} | 70 +++-
.../hadoop/hdds/server/http/ProfileServlet.java | 2 +-
.../hadoop/hdds/utils/DBCheckpointMetrics.java | 2 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 2 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 5 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +-
.../apache/hadoop/hdds/utils/TransactionInfo.java | 2 +-
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 4 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 4 +-
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 1 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +-
.../hadoop/hdds/utils/db/cache/CacheKey.java | 2 +-
.../hadoop/hdds/utils/db/cache/EpochEntry.java | 2 +-
.../x509/certificate/authority/MockCAStore.java | 2 +-
.../client/TestDefaultCertificateClient.java | 2 +-
.../x509/certificate/utils/TestCRLCodec.java | 2 +-
.../certificates/TestCertificateSignRequest.java | 4 +-
.../x509/certificates/TestRootCertificate.java | 2 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 2 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 35 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 5 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 10 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 40 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 6 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 4 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 2 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 20 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +-
.../interface-client/src/main/proto/hdds.proto | 7 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 11 +-
.../src/main/proto/ScmServerSecurityProtocol.proto | 8 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 2 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../container/AbstractContainerReportHandler.java | 44 +-
.../hdds/scm/container/ContainerManager.java | 7 +
.../hdds/scm/container/ContainerManagerImpl.java | 78 ++--
.../hdds/scm/container/ContainerReplicaCount.java | 18 +-
.../hdds/scm/container/ContainerReportHandler.java | 141 +++++--
.../hdds/scm/container/ContainerStateManager.java | 10 +-
.../scm/container/ContainerStateManagerImpl.java | 24 +-
.../IncrementalContainerReportHandler.java | 18 +-
.../hdds/scm/container/ReplicationManager.java | 24 +-
.../balancer/AbstractFindTargetGreedy.java | 6 +-
.../scm/container/balancer/ContainerBalancer.java | 121 ++++--
.../balancer/ContainerBalancerConfiguration.java | 27 +-
.../balancer/ContainerBalancerMetrics.java | 139 ++++---
.../scm/container/balancer/FindSourceGreedy.java | 6 +-
.../ContainerPlacementPolicyFactory.java | 2 +-
.../algorithms/SCMContainerPlacementMetrics.java | 2 +-
.../algorithms/SCMContainerPlacementRackAware.java | 12 +-
.../container/placement/metrics/SCMMetrics.java | 2 +-
.../scm/container/states/ContainerStateMap.java | 30 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 2 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 6 +-
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 9 +-
.../hadoop/hdds/scm/ha/SCMServiceManager.java | 3 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 2 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 4 +-
.../hdds/scm/metadata/X509CertificateCodec.java | 2 +-
.../apache/hadoop/hdds/scm/node/CommandQueue.java | 2 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 9 +-
.../hdds/scm/node/NodeDecommissionManager.java | 32 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 20 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 20 +-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 4 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 42 +-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 8 +-
.../hdds/scm/node/SCMNodeStorageStatMap.java | 2 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 2 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 6 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 4 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 7 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 11 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +-
.../scm/pipeline/WritableContainerFactory.java | 2 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 28 ++
...inerLocationProtocolServerSideTranslatorPB.java | 8 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 4 +-
.../hdds/scm/safemode/DataNodeSafeModeRule.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 16 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 15 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 18 +-
.../hadoop/hdds/scm/server/SCMPolicyProvider.java | 5 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 18 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 2 +-
.../hdds/scm/server/StorageContainerManager.java | 63 ++-
.../scm/server/StorageContainerManagerStarter.java | 2 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +-
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 2 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 37 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 4 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 27 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 22 +-
.../scm/container/TestContainerManagerImpl.java | 20 +-
.../scm/container/TestContainerReportHandler.java | 28 +-
.../scm/container/TestContainerStateManager.java | 6 +-
.../TestIncrementalContainerReportHandler.java | 30 +-
.../hdds/scm/container/TestReplicationManager.java | 134 +++---
.../container/balancer/TestContainerBalancer.java | 96 +++--
.../TestSCMContainerPlacementRackAware.java | 2 +-
.../states/TestContainerReplicaCount.java | 5 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 10 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 26 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 2 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 2 +-
.../hdds/scm/node/TestContainerPlacement.java | 2 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 11 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 12 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 26 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 13 +-
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 5 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 4 +-
.../TestPipelineDatanodesIntersection.java | 4 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 16 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 4 +-
.../TestOneReplicaPipelineSafeModeRule.java | 8 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 8 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 6 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 54 +--
.../server/TestStorageContainerManagerStarter.java | 6 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 24 +-
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 2 +-
.../testutils/ReplicationNodeManagerMock.java | 16 +-
.../org/apache/ozone/test/GenericTestUtils.java | 18 +-
.../org/apache/ozone/test/LambdaTestUtils.java | 4 +-
.../scm/cli/ContainerBalancerStatusSubcommand.java | 2 +-
.../cli/ReplicationManagerStatusSubcommand.java | 2 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 2 +-
.../hdds/scm/cli/SafeModeExitSubcommand.java | 2 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 4 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 2 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 38 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 4 +-
.../scm/cli/datanode/RecommissionSubCommand.java | 2 +-
.../scm/cli/container/TestReportSubCommand.java | 12 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 6 +-
hadoop-ozone/client/pom.xml | 1 +
.../apache/hadoop/ozone/client/ObjectStore.java | 6 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 12 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 2 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 2 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 2 +-
.../client/checksum/BaseFileChecksumHelper.java | 9 +-
.../checksum/ReplicatedFileChecksumHelper.java | 6 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 6 +-
.../client/io/BlockOutputStreamEntryPool.java | 1 -
.../client/io/MultipartCryptoKeyInputStream.java | 4 +
.../ozone/client/protocol/ClientProtocol.java | 4 +-
.../hadoop/ozone/client/rpc/OzoneKMSUtil.java | 4 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 28 +-
.../hadoop/ozone/client/TestHddsClientUtils.java | 4 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 18 +-
.../hadoop/ozone/client/rpc/TestOzoneKMSUtil.java | 3 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 4 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 8 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 6 +-
.../apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 10 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 1 +
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 7 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 2 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 4 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 6 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 4 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 2 +-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 6 +-
.../hadoop/ozone/om/helpers/ServiceInfo.java | 2 +-
.../hadoop/ozone/om/helpers/WithObjectID.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 21 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 51 ++-
.../apache/hadoop/ozone/security/acl/OzoneObj.java | 3 +-
.../hadoop/ozone/security/acl/OzoneObjInfo.java | 2 +-
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 2 +-
.../org/apache/hadoop/ozone/util/RadixTree.java | 2 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 18 +-
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 10 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 2 +-
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 2 +-
.../ozone/security/TestGDPRSymmetricKey.java | 2 +-
.../ozone/security/acl/TestOzoneObjInfo.java | 9 +-
.../apache/hadoop/ozone/util/TestRadixTree.java | 4 +-
hadoop-ozone/dev-support/checks/_lib.sh | 2 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 3 +-
hadoop-ozone/dev-support/checks/bats.sh | 8 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 3 +
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/compatibility/.env | 1 +
.../main/compose/compatibility/docker-compose.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 1 +
.../src/main/compose/ozone-csi/docker-compose.yaml | 8 +-
hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 1 +
.../src/main/compose/ozone-ha/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone-mr/hadoop27/.env | 1 +
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 1 +
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop32/.env | 1 +
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop33/.env | 1 +
.../compose/ozone-mr/hadoop33/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-om-ha/.env | 1 +
.../dist/src/main/compose/ozone-om-ha/Dockerfile | 3 +-
.../dist/src/main/compose/ozone-om-prepare/.env | 6 +-
.../compose/ozone-om-prepare/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone-topology/.env | 1 +
.../compose/ozone-topology/docker-compose.yaml | 16 +-
hadoop-ozone/dist/src/main/compose/ozone/.env | 1 +
.../src/main/compose/ozone/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone/freon-ockg.yaml | 2 +-
.../dist/src/main/compose/ozone/freon-rk.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 6 -
.../dist/src/main/compose/ozoneblockade/.env | 1 +
.../main/compose/ozoneblockade/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozones3-haproxy/.env | 1 +
.../compose/ozones3-haproxy/docker-compose.yaml | 12 +-
.../dist/src/main/compose/ozonescripts/.env | 1 +
.../dist/src/main/compose/ozonescripts/Dockerfile | 3 +-
.../main/compose/ozonescripts/docker-compose.yaml | 3 +
.../dist/src/main/compose/ozonesecure-ha/.env | 1 +
.../compose/ozonesecure-ha/docker-compose.yaml | 22 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 1 +
.../compose/ozonesecure-mr/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozonesecure-mr/test.sh | 3 -
.../dist/src/main/compose/ozonesecure/.env | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 10 +-
.../src/main/compose/ozonesecure/docker-config | 4 +
.../dist/src/main/compose/ozonesecure/test.sh | 2 +-
hadoop-ozone/dist/src/main/compose/restart/.env | 2 +-
.../src/main/compose/restart/docker-compose.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 14 +-
.../dist/src/main/compose/upgrade/compose/ha/.env | 1 +
.../main/compose/upgrade/compose/ha/docker-config | 4 +-
.../src/main/compose/upgrade/compose/non-ha/.env | 1 +
.../dist/src/main/compose/upgrade/testlib.sh | 1 -
hadoop-ozone/dist/src/main/compose/xcompat/.env | 1 +
.../dist/src/main/compose/xcompat/clients.yaml | 20 +-
.../dist/src/main/compose/xcompat/new-cluster.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 17 +-
.../k8s/definitions/ozone-csi/csi-controller.yaml | 2 +-
.../main/k8s/examples/getting-started/Flekszible | 2 +
.../examples/getting-started/config-configmap.yaml | 1 +
.../examples/getting-started/kustomization.yaml} | 20 +-
.../dist/src/main/k8s/examples/minikube/Flekszible | 2 +
.../k8s/examples/minikube/config-configmap.yaml | 1 +
.../examples/minikube/kustomization.yaml} | 20 +-
.../src/main/k8s/examples/ozone-dev/Flekszible | 1 +
.../k8s/examples/ozone-dev/config-configmap.yaml | 1 +
.../main/k8s/examples/ozone-dev/kustomization.yaml | 31 +-
.../dist/src/main/k8s/examples/ozone-ha/Flekszible | 3 +
.../examples/ozone-ha/kustomization.yaml} | 16 +-
.../dist/src/main/k8s/examples/ozone/Flekszible | 3 +-
.../main/k8s/examples/ozone/config-configmap.yaml | 1 +
.../examples/ozone/kustomization.yaml} | 16 +-
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 3 +-
.../src/main/smoketest/compatibility/read.robot | 9 +-
.../{kinit.robot => compatibility/setup.robot} | 16 +-
.../src/main/smoketest/compatibility/write.robot | 7 +-
.../dist/src/main/smoketest/mapreduce.robot | 1 +
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 1 +
.../src/main/smoketest/s3/MultipartUpload.robot | 21 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 36 ++
hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh | 3 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 10 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 2 +-
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 2 +-
.../hadoop/ozone/loadgenerators/LoadBucket.java | 2 +-
.../hadoop/ozone/insight/TestBaseInsightPoint.java | 7 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 14 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 2 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 42 +-
.../fs/ozone/TestOzoneFileSystemWithLinks.java | 14 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 4 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 36 +-
.../fs/ozone/contract/ITestOzoneContractUtils.java | 4 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 4 +-
.../TestContainerStateManagerIntegration.java | 28 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 2 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 2 +-
.../TestRatisPipelineCreateAndDestroy.java | 2 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 6 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 6 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 2 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 14 +-
.../hadoop/ozone/MiniOzoneClusterProvider.java | 4 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +-
...OutputUtil.java => StandardOutputTestBase.java} | 2 +-
.../ozone/TestContainerBalancerOperations.java | 4 +-
.../hadoop/ozone/TestContainerOperations.java | 2 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +-
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 15 +-
.../hadoop/ozone/TestStorageContainerManager.java | 5 +-
.../ozone/client/CertificateClientTestImpl.java | 4 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 4 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 14 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 +-
.../client/rpc/TestFailureHandlingByClient.java | 116 +++++-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 5 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 4 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 38 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 10 +-
.../rpc/TestOzoneClientRetriesOnExceptions.java | 4 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 60 +--
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 21 +-
.../TestOzoneRpcClientWithKeyLatestVersion.java | 2 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 2 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 6 +-
.../ozone/client/rpc/TestWatchForCommit.java | 2 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 8 +-
.../apache/hadoop/ozone/container/TestHelper.java | 8 +-
.../TestCloseContainerByPipeline.java | 2 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
...ler.java => TestRefreshVolumeUsageHandler.java} | 121 +++---
.../transport/server/ratis/TestCSMMetrics.java | 3 +-
.../container/metrics/TestContainerMetrics.java | 4 +-
.../container/ozoneimpl/TestOzoneContainer.java | 27 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 6 +-
.../container/server/TestContainerServer.java | 2 +-
.../server/TestSecureContainerServer.java | 16 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 2 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 26 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 16 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 27 +-
.../ozone/om/TestContainerReportWithKeys.java | 4 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 12 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 14 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 +-
.../ozone/om/TestOmStartupSlvLessThanMlv.java | 2 +-
.../ozone/om/TestOzoneManagerConfiguration.java | 4 +-
.../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 8 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 2 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 47 ++-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 3 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +-
.../ozone/recon/TestReconWithOzoneManager.java | 16 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 8 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 2 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 4 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 6 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 10 +-
.../scm/node/TestDecommissionAndMaintenance.java | 24 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 2 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 4 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 22 +-
.../src/main/proto/OmClientProtocol.proto | 1 +
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 2 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 58 +--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 28 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 22 +-
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 2 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 13 +-
.../hadoop/ozone/om/OzoneManagerPrepareState.java | 4 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 2 +-
.../hadoop/ozone/om/OzonePrefixPathImpl.java | 18 +
.../hadoop/ozone/om/S3SecretManagerImpl.java | 2 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 12 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 12 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 4 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 6 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 21 +-
.../om/request/bucket/OMBucketCreateRequest.java | 18 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 9 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 6 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 2 +-
.../om/request/file/OMDirectoryCreateRequest.java | 2 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 5 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 2 +-
.../ozone/om/request/file/OMFileRequest.java | 10 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 2 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 2 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 4 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 8 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +-
.../multipart/S3MultipartUploadAbortRequest.java | 2 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 2 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 2 +-
.../volume/acl/OMVolumeRemoveAclRequest.java | 2 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 2 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 3 +-
.../om/response/key/OMKeysRenameResponse.java | 10 +
.../multipart/S3MultipartUploadAbortResponse.java | 6 +-
.../S3MultipartUploadCommitPartResponse.java | 6 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 1 +
.../OzoneDelegationTokenSecretManager.java | 2 +-
.../hadoop/ozone/security/OzoneSecretStore.java | 4 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 7 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 2 +-
.../apache/hadoop/ozone/om/TestOMDBDefinition.java | 2 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 14 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 4 +-
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 4 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 2 +-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 4 +
....java => TestOMBucketDeleteRequestWithFSO.java} | 72 +---
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../om/request/key/TestOMKeyCommitRequest.java | 4 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 2 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 101 ++++-
.../key/TestOMKeyPurgeRequestAndResponse.java | 2 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 2 +-
.../TestS3MultipartUploadCompleteRequest.java | 2 +-
.../request/volume/TestOMVolumeCreateRequest.java | 2 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 2 +-
.../ozone/om/response/TestCleanupTableInfo.java | 4 +-
.../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 2 +-
.../om/upgrade/TestOzoneManagerPrepareState.java | 2 +-
.../ozone/security/TestAWSV4AuthValidator.java | 2 +-
.../TestOzoneDelegationTokenSecretManager.java | 6 +-
.../ozone/security/TestOzoneTokenIdentifier.java | 6 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 10 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 16 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 9 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 18 +
.../fs/ozone/BasicRootedOzoneFileSystem.java | 3 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 +
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 24 +-
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 2 +-
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 59 +++
hadoop-ozone/ozonefs-shaded/pom.xml | 2 +-
.../org/hadoop/ozone/recon/codegen/SqlDbUtils.java | 4 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 125 ++++++
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 7 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 29 ++
.../ozone/recon/api/ClusterStateEndpoint.java | 15 +-
.../ozone/recon/api/MetricsProxyEndpoint.java | 4 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 2 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 2 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 4 +-
.../recon/metrics/OzoneManagerSyncMetrics.java | 12 +
.../recon/metrics/ReconTaskStatusMetrics.java | 83 ++++
.../ozone/recon/scm/ReconContainerManager.java | 4 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 18 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 2 +-
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 61 ++-
.../scm/ReconStorageContainerManagerFacade.java | 11 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 2 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 68 ++-
.../impl/StorageContainerServiceProviderImpl.java | 16 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 4 +-
.../hadoop/ozone/recon/tasks/TableCountTask.java | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +
.../ozone/recon/api/TestOpenContainerCount.java | 8 +-
.../ozone/recon/api/TestTaskStatusService.java | 2 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 2 +-
.../TestUtilizationSchemaDefinition.java | 2 +-
.../scm/AbstractReconContainerManagerTest.java | 3 +-
.../ozone/recon/scm/TestReconNodeManager.java | 12 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 3 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 88 +++-
.../impl/TestReconNamespaceSummaryManagerImpl.java | 6 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 2 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 4 +-
.../ozone/recon/tasks/TestTableCountTask.java | 2 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 19 +-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 8 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 58 +--
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 24 +-
.../ozone/s3/endpoint/ListBucketResponse.java | 6 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 108 +++--
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 6 +-
.../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 4 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 20 +-
.../hadoop/ozone/s3/signature/Credential.java | 2 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 2 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 39 +-
.../ozone/s3/TestVirtualHostStyleFilter.java | 2 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 2 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 6 +-
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 6 +-
.../admin/scm/FinalizeUpgradeCommandUtil.java | 4 +-
.../hadoop/ozone/audit/parser/AuditParser.java | 2 +-
.../ozone/audit/parser/common/DatabaseHelper.java | 16 +-
.../parser/handler/TemplateCommandHandler.java | 2 +-
.../ozone/audit/parser/model/AuditEntry.java | 22 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 6 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 8 +-
.../apache/hadoop/ozone/debug/PrefixParser.java | 2 +-
.../apache/hadoop/ozone/debug/ReadReplicas.java | 19 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 2 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 12 +-
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 4 +-
.../hadoop/ozone/freon/HadoopFsGenerator.java | 2 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 2 +-
.../GenerateOzoneRequiredConfigurations.java | 2 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 4 +-
.../ozone/shell/bucket/CreateBucketHandler.java | 2 +-
.../hadoop/ozone/audit/parser/TestAuditParser.java | 2 +-
.../hadoop/ozone/conf/TestGetConfOptions.java | 4 +-
.../apache/hadoop/ozone/freon/TestProgressBar.java | 6 +-
.../TestGenerateOzoneRequiredConfigurations.java | 4 +-
.../org/apache/hadoop/test/OzoneTestDriver.java | 6 +-
pom.xml | 30 +-
719 files changed, 6511 insertions(+), 3535 deletions(-)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerInspector.java
copy hadoop-hdds/container-service/src/{test/java/org/apache/hadoop/ozone/container/common/report/package-info.java => main/java/org/apache/hadoop/ozone/container/common/report/IncrementalReportSender.java} (75%)
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerInspectorUtil.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/RefreshVolumeUsageCommand.java
copy hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/{TestKeyValueContainerCheck.java => TestKeyValueContainerIntegrityChecks.java} (52%)
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
create mode 100644 hadoop-hdds/docs/dev-support/bin/make_images_responsive.py
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/{OMCertificateClient.java => CommonCertificateClient.java} (51%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/BadRequestExceptionMapper.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java (57%)
rename hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/{FixedThreadPoolExecutor.java => FixedThreadPoolWithAffinityExecutor.java} (59%)
copy hadoop-ozone/dist/src/main/{compose/common/grafana/provisioning/dashboards/dashboards.yml => k8s/examples/getting-started/kustomization.yaml} (71%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/{compose/common/grafana/provisioning/dashboards/dashboards.yml => k8s/examples/minikube/kustomization.yaml} (71%)
mode change 100755 => 100644
copy hadoop-hdds/server-scm/src/test/resources/rack-mapping => hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/kustomization.yaml (56%)
copy hadoop-ozone/dist/src/main/{compose/common/grafana/provisioning/dashboards/dashboards.yml => k8s/examples/ozone-ha/kustomization.yaml} (78%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/{compose/common/grafana/provisioning/dashboards/dashboards.yml => k8s/examples/ozone/kustomization.yaml} (78%)
mode change 100755 => 100644
copy hadoop-ozone/dist/src/main/smoketest/{kinit.robot => compatibility/setup.robot} (75%)
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{TestStandardOutputUtil.java => StandardOutputTestBase.java} (98%)
copy hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/{TestCloseContainerHandler.java => TestRefreshVolumeUsageHandler.java} (55%)
copy hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/{TestOMBucketDeleteRequest.java => TestOMBucketDeleteRequestWithFSO.java} (54%)
create mode 100644 hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneClientUtils.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/ReconTaskStatusMetrics.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
[ozone] 01/01: Merge remote-tracking branch 'origin/master' into HDDS-6030
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-6030
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 70a9b9c160f2779be76026e82358588f8f9457f8
Merge: c48b559 b83c1f9
Author: Doroszlai, Attila <ad...@apache.org>
AuthorDate: Thu Mar 3 12:20:02 2022 +0100
Merge remote-tracking branch 'origin/master' into HDDS-6030
.github/workflows/post-commit.yml | 8 +-
.gitignore | 3 +-
dev-support/ci/selective_ci_checks.bats | 70 +++-
dev-support/ci/selective_ci_checks.sh | 76 +++-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 38 ++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 8 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 2 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 2 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 2 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +-
.../java/org/apache/hadoop/hdds/StringUtils.java | 2 +-
.../hadoop/hdds/annotation/InterfaceAudience.java | 6 +-
.../org/apache/hadoop/hdds/client/OzoneQuota.java | 2 +-
.../org/apache/hadoop/hdds/client/QuotaList.java | 11 +-
.../hadoop/hdds/client/ReplicationFactor.java | 2 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 2 +-
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 2 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 4 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 2 +-
.../RequestTypeDependentRetryPolicyCreator.java | 6 +-
.../org/apache/hadoop/hdds/recon/ReconConfig.java | 2 +-
.../hadoop/hdds/scm/ByteStringConversion.java | 2 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 2 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 21 +-
.../scm/container/ReplicationManagerReport.java | 2 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 9 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 16 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 28 +-
.../org/apache/hadoop/hdds/scm/net/NodeSchema.java | 4 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 8 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 8 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 8 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 2 +-
.../hadoop/hdds/utils/ResourceSemaphore.java | 10 +-
.../org/apache/hadoop/hdds/utils/UniqueId.java | 2 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 11 +-
.../hadoop/ozone/audit/AuditEventStatus.java | 2 +-
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 4 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 2 +-
.../org/apache/hadoop/ozone/audit/AuditMarker.java | 4 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 10 +-
.../hadoop/ozone/common/ChecksumByteBuffer.java | 2 +-
.../ozone/common/IncrementalChunkBuffer.java | 13 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 8 +-
.../ozone/common/ha/ratis/RatisSnapshotInfo.java | 2 +-
.../ozone/container/common/helpers/BlockData.java | 4 +-
.../helpers/ContainerCommandRequestPBHelper.java | 10 +-
.../java/org/apache/hadoop/ozone/lease/Lease.java | 8 +-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 26 ++
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 4 +-
.../hadoop/hdds/client/TestReplicationConfig.java | 2 +-
.../java/org/apache/hadoop/hdds/fs/TestDU.java | 2 +-
.../ratis/TestContainerCommandRequestMessage.java | 6 +-
.../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 4 +-
.../hdds/scm/net/TestNetworkTopologyImpl.java | 20 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 2 +-
.../hadoop/hdds/utils/MockGatheringChannel.java | 2 +-
.../hadoop/hdds/utils/TestResourceSemaphore.java | 6 +-
.../org/apache/hadoop/ozone/audit/DummyEntity.java | 2 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 4 +-
.../apache/hadoop/ozone/common/TestChecksum.java | 2 +-
.../hadoop/ozone/common/TestChunkBuffer.java | 20 +-
.../hadoop/ozone/common/TestStateMachine.java | 4 +-
.../ozone/container/ContainerTestHelper.java | 70 +---
.../hadoop/ozone/lease/TestLeaseManager.java | 2 +-
.../TestDefaultUpgradeFinalizationExecutor.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 27 +-
.../container/common/helpers/ContainerMetrics.java | 6 +-
.../container/common/helpers/ContainerUtils.java | 2 +-
.../ozone/container/common/impl/ContainerData.java | 46 +-
.../ozone/container/common/impl/ContainerSet.java | 16 +-
.../container/common/impl/HddsDispatcher.java | 15 +-
.../common/impl/OpenContainerBlockMap.java | 2 +-
.../common/interfaces/ContainerInspector.java | 72 ++++
.../ozone/container/common/interfaces/Handler.java | 12 +-
.../common/report/ContainerReportPublisher.java | 5 +-
.../common/report/IncrementalReportSender.java} | 26 +-
.../common/statemachine/DatanodeConfiguration.java | 2 +-
.../common/statemachine/DatanodeStateMachine.java | 14 +-
.../common/statemachine/StateContext.java | 143 ++++---
.../commandhandler/CommandDispatcher.java | 2 +-
.../SetNodeOperationalStateCommandHandler.java | 11 -
.../common/states/datanode/InitDatanodeState.java | 9 +-
.../states/datanode/RunningDatanodeState.java | 8 +-
.../states/endpoint/HeartbeatEndpointTask.java | 23 +-
.../states/endpoint/VersionEndpointTask.java | 2 +-
.../common/transport/server/ratis/CSMMetrics.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 11 +-
.../transport/server/ratis/XceiverServerRatis.java | 9 +-
.../common/utils/ContainerInspectorUtil.java | 87 ++++
.../container/common/utils/HddsVolumeUtil.java | 2 +-
.../container/common/volume/AbstractFuture.java | 13 +-
.../container/common/volume/MutableVolumeSet.java | 2 +-
.../container/common/volume/StorageVolume.java | 2 +-
.../ozone/container/common/volume/VolumeUsage.java | 2 +-
.../container/keyvalue/KeyValueContainer.java | 82 +++-
.../container/keyvalue/KeyValueContainerCheck.java | 6 +-
.../container/keyvalue/KeyValueContainerData.java | 12 +-
.../KeyValueContainerMetadataInspector.java | 463 +++++++++++++++++++++
.../ozone/container/keyvalue/KeyValueHandler.java | 101 ++---
.../container/keyvalue/TarContainerPacker.java | 55 ++-
.../container/keyvalue/helpers/ChunkUtils.java | 4 +
.../helpers/KeyValueContainerLocationUtil.java | 2 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 23 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 110 +++--
.../keyvalue/impl/ChunkManagerDispatcher.java | 18 +-
.../background/BlockDeletingService.java | 112 +++--
.../container/metadata/AbstractDatanodeStore.java | 8 +-
.../container/ozoneimpl/ContainerController.java | 14 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 6 +-
.../ozoneimpl/ContainerMetadataScanner.java | 2 +-
.../ContainerMetadataScrubberMetrics.java | 2 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 6 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 40 +-
.../replication/GrpcReplicationClient.java | 2 +-
.../container/replication/MeasuredReplicator.java | 3 +-
.../container/stream/DirstreamClientHandler.java | 2 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 2 +-
.../upgrade/VersionedDatanodeFeatures.java | 2 +-
.../commands/RefreshVolumeUsageCommand.java | 57 +++
.../ozone/protocol/commands/ReregisterCommand.java | 2 +-
.../hadoop/ozone/container/common/ScmTestMock.java | 42 +-
.../container/common/TestBlockDeletingService.java | 13 +-
.../ozone/container/common/TestContainerCache.java | 2 +-
.../container/common/TestDatanodeStateMachine.java | 5 -
.../common/TestKeyValueContainerData.java | 6 +-
.../TestSchemaOneBackwardsCompatibility.java | 38 +-
.../container/common/helpers/TestBlockData.java | 6 +-
.../common/helpers/TestDatanodeVersionFile.java | 14 +-
.../common/impl/TestContainerDataYaml.java | 2 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 2 +-
.../common/impl/TestContainerPersistence.java | 47 ---
.../container/common/impl/TestContainerSet.java | 12 +-
.../container/common/impl/TestHddsDispatcher.java | 7 +-
.../container/common/interfaces/TestHandler.java | 2 +-
.../common/report/TestReportPublisher.java | 2 +-
.../common/statemachine/TestStateContext.java | 71 +++-
.../TestCloseContainerCommandHandler.java | 4 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 2 +-
.../common/volume/TestStorageVolumeChecker.java | 12 +-
.../container/common/volume/TestVolumeSet.java | 2 +-
.../keyvalue/TestKeyValueBlockIterator.java | 24 +-
.../container/keyvalue/TestKeyValueContainer.java | 69 ++-
.../keyvalue/TestKeyValueContainerCheck.java | 160 +------
...a => TestKeyValueContainerIntegrityChecks.java} | 160 ++-----
.../TestKeyValueContainerMetadataInspector.java | 360 ++++++++++++++++
.../container/keyvalue/TestKeyValueHandler.java | 23 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 14 -
.../keyvalue/impl/CommonChunkManagerTestCases.java | 5 +-
.../keyvalue/impl/TestBlockManagerImpl.java | 57 +--
.../container/ozoneimpl/TestContainerReader.java | 10 +-
.../container/ozoneimpl/TestOzoneContainer.java | 6 +-
.../replication/TestGrpcOutputStream.java | 8 +-
.../replication/TestMeasuredReplicator.java | 15 +
.../replication/TestReplicationSupervisor.java | 2 +-
.../testutils/BlockDeletingServiceTestImpl.java | 2 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 6 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 2 +
hadoop-hdds/docs/content/concept/Containers.md | 3 +-
hadoop-hdds/docs/content/concept/Datanodes.md | 4 +-
hadoop-hdds/docs/content/concept/OzoneManager.md | 6 +-
hadoop-hdds/docs/content/concept/Recon.md | 9 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 4 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 68 ++-
hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.md | 2 +-
hadoop-hdds/docs/content/interface/O3fs.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Ofs.md | 2 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 48 ++-
.../docs/content/security/SecuringTDE.zh.md | 4 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 +++
.../themes/ozonedoc/layouts/shortcodes/image.html | 2 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 12 +
.../SCMSecurityProtocolClientSideTranslatorPB.java | 39 +-
...inerLocationProtocolClientSideTranslatorPB.java | 4 +-
.../scm/update/client/CRLClientUpdateHandler.java | 2 +-
.../hdds/scm/update/client/ClientCRLStore.java | 4 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 2 +-
.../scm/update/client/UpdateServiceConfig.java | 5 +-
.../x509/certificate/authority/BaseApprover.java | 2 +-
.../certificate/authority/DefaultCAServer.java | 6 +-
.../authority/PKIProfiles/DefaultCAProfile.java | 2 +-
.../authority/PKIProfiles/DefaultProfile.java | 32 +-
...ateClient.java => CommonCertificateClient.java} | 116 +++---
.../client/DefaultCertificateClient.java | 32 +-
.../certificate/client/OMCertificateClient.java | 84 +---
.../client/ReconCertificateClient.java} | 40 +-
.../certificates/utils/CertificateSignRequest.java | 10 +-
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 2 +-
.../hdds/security/x509/crl/CRLInfoCodec.java | 2 +-
...va => FixedThreadPoolWithAffinityExecutor.java} | 70 +++-
.../hadoop/hdds/server/http/ProfileServlet.java | 2 +-
.../hadoop/hdds/utils/DBCheckpointMetrics.java | 2 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 2 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 5 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +-
.../apache/hadoop/hdds/utils/TransactionInfo.java | 2 +-
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 4 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 4 +-
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 9 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 1 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +-
.../hadoop/hdds/utils/db/cache/CacheKey.java | 2 +-
.../hadoop/hdds/utils/db/cache/EpochEntry.java | 2 +-
.../x509/certificate/authority/MockCAStore.java | 2 +-
.../client/TestDefaultCertificateClient.java | 2 +-
.../x509/certificate/utils/TestCRLCodec.java | 2 +-
.../certificates/TestCertificateSignRequest.java | 4 +-
.../x509/certificates/TestRootCertificate.java | 2 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 2 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 35 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 5 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 10 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 40 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 6 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 4 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 2 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 20 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +-
.../interface-client/src/main/proto/hdds.proto | 7 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 11 +-
.../src/main/proto/ScmServerSecurityProtocol.proto | 8 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 2 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../container/AbstractContainerReportHandler.java | 44 +-
.../hdds/scm/container/ContainerManager.java | 7 +
.../hdds/scm/container/ContainerManagerImpl.java | 78 ++--
.../hdds/scm/container/ContainerReplicaCount.java | 18 +-
.../hdds/scm/container/ContainerReportHandler.java | 141 +++++--
.../hdds/scm/container/ContainerStateManager.java | 10 +-
.../scm/container/ContainerStateManagerImpl.java | 24 +-
.../IncrementalContainerReportHandler.java | 18 +-
.../hdds/scm/container/ReplicationManager.java | 24 +-
.../balancer/AbstractFindTargetGreedy.java | 6 +-
.../scm/container/balancer/ContainerBalancer.java | 121 ++++--
.../balancer/ContainerBalancerConfiguration.java | 27 +-
.../balancer/ContainerBalancerMetrics.java | 139 ++++---
.../scm/container/balancer/FindSourceGreedy.java | 6 +-
.../ContainerPlacementPolicyFactory.java | 2 +-
.../algorithms/SCMContainerPlacementMetrics.java | 2 +-
.../algorithms/SCMContainerPlacementRackAware.java | 12 +-
.../container/placement/metrics/SCMMetrics.java | 2 +-
.../scm/container/states/ContainerStateMap.java | 30 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 2 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 6 +-
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 9 +-
.../hadoop/hdds/scm/ha/SCMServiceManager.java | 3 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 2 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 4 +-
.../hdds/scm/metadata/X509CertificateCodec.java | 2 +-
.../apache/hadoop/hdds/scm/node/CommandQueue.java | 2 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 9 +-
.../hdds/scm/node/NodeDecommissionManager.java | 32 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 20 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 20 +-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 4 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 42 +-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 8 +-
.../hdds/scm/node/SCMNodeStorageStatMap.java | 2 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 2 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 6 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 4 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 7 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 11 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +-
.../scm/pipeline/WritableContainerFactory.java | 2 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 28 ++
...inerLocationProtocolServerSideTranslatorPB.java | 8 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 4 +-
.../hdds/scm/safemode/DataNodeSafeModeRule.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 16 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 15 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 18 +-
.../hadoop/hdds/scm/server/SCMPolicyProvider.java | 5 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 18 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 2 +-
.../hdds/scm/server/StorageContainerManager.java | 63 ++-
.../scm/server/StorageContainerManagerStarter.java | 2 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +-
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 2 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 37 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 4 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 27 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 22 +-
.../scm/container/TestContainerManagerImpl.java | 20 +-
.../scm/container/TestContainerReportHandler.java | 28 +-
.../scm/container/TestContainerStateManager.java | 6 +-
.../TestIncrementalContainerReportHandler.java | 30 +-
.../hdds/scm/container/TestReplicationManager.java | 134 +++---
.../container/balancer/TestContainerBalancer.java | 96 +++--
.../TestSCMContainerPlacementRackAware.java | 2 +-
.../states/TestContainerReplicaCount.java | 5 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 10 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 26 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 2 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 2 +-
.../hdds/scm/node/TestContainerPlacement.java | 2 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 11 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 12 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 26 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 13 +-
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 5 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 4 +-
.../TestPipelineDatanodesIntersection.java | 4 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 16 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 4 +-
.../TestOneReplicaPipelineSafeModeRule.java | 8 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 8 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 6 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 54 +--
.../server/TestStorageContainerManagerStarter.java | 6 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 24 +-
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 2 +-
.../testutils/ReplicationNodeManagerMock.java | 16 +-
.../org/apache/ozone/test/GenericTestUtils.java | 18 +-
.../org/apache/ozone/test/LambdaTestUtils.java | 4 +-
.../scm/cli/ContainerBalancerStatusSubcommand.java | 2 +-
.../cli/ReplicationManagerStatusSubcommand.java | 2 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 2 +-
.../hdds/scm/cli/SafeModeExitSubcommand.java | 2 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 4 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 2 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 38 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 4 +-
.../scm/cli/datanode/RecommissionSubCommand.java | 2 +-
.../scm/cli/container/TestReportSubCommand.java | 12 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 6 +-
hadoop-ozone/client/pom.xml | 1 +
.../apache/hadoop/ozone/client/ObjectStore.java | 6 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 12 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 2 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 2 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 2 +-
.../client/checksum/BaseFileChecksumHelper.java | 9 +-
.../checksum/ReplicatedFileChecksumHelper.java | 6 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 6 +-
.../client/io/BlockOutputStreamEntryPool.java | 1 -
.../client/io/MultipartCryptoKeyInputStream.java | 4 +
.../ozone/client/protocol/ClientProtocol.java | 4 +-
.../hadoop/ozone/client/rpc/OzoneKMSUtil.java | 4 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 28 +-
.../hadoop/ozone/client/TestHddsClientUtils.java | 4 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 18 +-
.../hadoop/ozone/client/rpc/TestOzoneKMSUtil.java | 3 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 4 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 8 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 6 +-
.../apache/hadoop/ozone/om/helpers/DBUpdates.java | 10 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 10 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 1 +
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 7 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 2 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 4 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 6 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 4 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 2 +-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 6 +-
.../hadoop/ozone/om/helpers/ServiceInfo.java | 2 +-
.../hadoop/ozone/om/helpers/WithObjectID.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 21 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 51 ++-
.../apache/hadoop/ozone/security/acl/OzoneObj.java | 3 +-
.../hadoop/ozone/security/acl/OzoneObjInfo.java | 2 +-
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 2 +-
.../org/apache/hadoop/ozone/util/RadixTree.java | 2 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 18 +-
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 10 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 2 +-
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 2 +-
.../ozone/security/TestGDPRSymmetricKey.java | 2 +-
.../ozone/security/acl/TestOzoneObjInfo.java | 9 +-
.../apache/hadoop/ozone/util/TestRadixTree.java | 4 +-
hadoop-ozone/dev-support/checks/_lib.sh | 2 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 3 +-
hadoop-ozone/dev-support/checks/bats.sh | 8 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 3 +
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/compatibility/.env | 1 +
.../main/compose/compatibility/docker-compose.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 1 +
.../src/main/compose/ozone-csi/docker-compose.yaml | 8 +-
hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 1 +
.../src/main/compose/ozone-ha/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone-mr/hadoop27/.env | 1 +
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 1 +
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop32/.env | 1 +
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop33/.env | 1 +
.../compose/ozone-mr/hadoop33/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-om-ha/.env | 1 +
.../dist/src/main/compose/ozone-om-ha/Dockerfile | 3 +-
.../dist/src/main/compose/ozone-om-prepare/.env | 6 +-
.../compose/ozone-om-prepare/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone-topology/.env | 1 +
.../compose/ozone-topology/docker-compose.yaml | 16 +-
hadoop-ozone/dist/src/main/compose/ozone/.env | 1 +
.../src/main/compose/ozone/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone/freon-ockg.yaml | 2 +-
.../dist/src/main/compose/ozone/freon-rk.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 6 -
.../dist/src/main/compose/ozoneblockade/.env | 1 +
.../main/compose/ozoneblockade/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozones3-haproxy/.env | 1 +
.../compose/ozones3-haproxy/docker-compose.yaml | 12 +-
.../dist/src/main/compose/ozonescripts/.env | 1 +
.../dist/src/main/compose/ozonescripts/Dockerfile | 3 +-
.../main/compose/ozonescripts/docker-compose.yaml | 3 +
.../dist/src/main/compose/ozonesecure-ha/.env | 1 +
.../compose/ozonesecure-ha/docker-compose.yaml | 22 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 1 +
.../compose/ozonesecure-mr/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozonesecure-mr/test.sh | 3 -
.../dist/src/main/compose/ozonesecure/.env | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 10 +-
.../src/main/compose/ozonesecure/docker-config | 4 +
.../dist/src/main/compose/ozonesecure/test.sh | 2 +-
hadoop-ozone/dist/src/main/compose/restart/.env | 2 +-
.../src/main/compose/restart/docker-compose.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 14 +-
.../dist/src/main/compose/upgrade/compose/ha/.env | 1 +
.../main/compose/upgrade/compose/ha/docker-config | 4 +-
.../src/main/compose/upgrade/compose/non-ha/.env | 1 +
.../dist/src/main/compose/upgrade/testlib.sh | 1 -
hadoop-ozone/dist/src/main/compose/xcompat/.env | 1 +
.../dist/src/main/compose/xcompat/clients.yaml | 20 +-
.../dist/src/main/compose/xcompat/new-cluster.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 17 +-
.../k8s/definitions/ozone-csi/csi-controller.yaml | 2 +-
.../main/k8s/examples/getting-started/Flekszible | 2 +
.../examples/getting-started/config-configmap.yaml | 1 +
.../examples/getting-started/kustomization.yaml} | 20 +-
.../dist/src/main/k8s/examples/minikube/Flekszible | 2 +
.../k8s/examples/minikube/config-configmap.yaml | 1 +
.../examples/minikube/kustomization.yaml} | 20 +-
.../src/main/k8s/examples/ozone-dev/Flekszible | 1 +
.../k8s/examples/ozone-dev/config-configmap.yaml | 1 +
.../examples/ozone-dev/kustomization.yaml} | 26 +-
.../dist/src/main/k8s/examples/ozone-ha/Flekszible | 3 +
.../examples/ozone-ha/kustomization.yaml} | 15 +-
.../dist/src/main/k8s/examples/ozone/Flekszible | 3 +-
.../main/k8s/examples/ozone/config-configmap.yaml | 1 +
.../.env => k8s/examples/ozone/kustomization.yaml} | 15 +-
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 3 +-
.../src/main/smoketest/compatibility/read.robot | 9 +-
.../compatibility/{write.robot => setup.robot} | 16 +-
.../src/main/smoketest/compatibility/write.robot | 7 +-
.../dist/src/main/smoketest/mapreduce.robot | 1 +
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 1 +
.../src/main/smoketest/s3/MultipartUpload.robot | 21 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 36 ++
hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh | 3 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 10 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 2 +-
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 2 +-
.../hadoop/ozone/loadgenerators/LoadBucket.java | 2 +-
.../hadoop/ozone/insight/TestBaseInsightPoint.java | 7 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 14 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 2 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 42 +-
.../fs/ozone/TestOzoneFileSystemWithLinks.java | 14 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 4 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 36 +-
.../fs/ozone/contract/ITestOzoneContractUtils.java | 4 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 4 +-
.../TestContainerStateManagerIntegration.java | 28 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 2 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 2 +-
.../TestRatisPipelineCreateAndDestroy.java | 2 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 6 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 6 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 2 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 14 +-
.../hadoop/ozone/MiniOzoneClusterProvider.java | 4 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +-
...OutputUtil.java => StandardOutputTestBase.java} | 2 +-
.../ozone/TestContainerBalancerOperations.java | 4 +-
.../hadoop/ozone/TestContainerOperations.java | 2 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +-
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 15 +-
.../hadoop/ozone/TestStorageContainerManager.java | 5 +-
.../ozone/client/CertificateClientTestImpl.java | 4 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 4 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 14 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 +-
.../client/rpc/TestFailureHandlingByClient.java | 116 +++++-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 5 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 4 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 38 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 10 +-
.../rpc/TestOzoneClientRetriesOnExceptions.java | 4 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 60 +--
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 21 +-
.../TestOzoneRpcClientWithKeyLatestVersion.java | 2 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 2 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 6 +-
.../ozone/client/rpc/TestWatchForCommit.java | 2 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 8 +-
.../apache/hadoop/ozone/container/TestHelper.java | 8 +-
.../TestCloseContainerByPipeline.java | 2 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
.../TestRefreshVolumeUsageHandler.java | 145 +++++++
.../transport/server/ratis/TestCSMMetrics.java | 3 +-
.../container/metrics/TestContainerMetrics.java | 4 +-
.../container/ozoneimpl/TestOzoneContainer.java | 27 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 6 +-
.../container/server/TestContainerServer.java | 2 +-
.../server/TestSecureContainerServer.java | 16 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 2 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 26 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 16 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 27 +-
.../ozone/om/TestContainerReportWithKeys.java | 4 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 12 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 14 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 +-
.../ozone/om/TestOmStartupSlvLessThanMlv.java | 2 +-
.../ozone/om/TestOzoneManagerConfiguration.java | 4 +-
.../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 8 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 2 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 47 ++-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 3 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +-
.../ozone/recon/TestReconWithOzoneManager.java | 16 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 8 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 2 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 4 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 6 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 10 +-
.../scm/node/TestDecommissionAndMaintenance.java | 24 +-
.../hadoop/ozone/scm/node/TestQueryNode.java | 2 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 4 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 22 +-
.../src/main/proto/OmClientProtocol.proto | 1 +
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 2 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 58 +--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 28 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 22 +-
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 2 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 13 +-
.../hadoop/ozone/om/OzoneManagerPrepareState.java | 4 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 2 +-
.../hadoop/ozone/om/OzonePrefixPathImpl.java | 18 +
.../hadoop/ozone/om/S3SecretManagerImpl.java | 2 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 12 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 12 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 4 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 6 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 21 +-
.../om/request/bucket/OMBucketCreateRequest.java | 18 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 9 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 6 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 2 +-
.../om/request/file/OMDirectoryCreateRequest.java | 2 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 5 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 2 +-
.../ozone/om/request/file/OMFileRequest.java | 10 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 2 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 2 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 4 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 8 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +-
.../multipart/S3MultipartUploadAbortRequest.java | 2 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 2 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 2 +-
.../volume/acl/OMVolumeRemoveAclRequest.java | 2 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 2 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 3 +-
.../om/response/key/OMKeysRenameResponse.java | 10 +
.../multipart/S3MultipartUploadAbortResponse.java | 6 +-
.../S3MultipartUploadCommitPartResponse.java | 6 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 1 +
.../OzoneDelegationTokenSecretManager.java | 2 +-
.../hadoop/ozone/security/OzoneSecretStore.java | 4 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 7 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 2 +-
.../apache/hadoop/ozone/om/TestOMDBDefinition.java | 2 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 14 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 4 +-
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 4 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 2 +-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 4 +
.../bucket/TestOMBucketDeleteRequestWithFSO.java | 76 ++++
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../om/request/key/TestOMKeyCommitRequest.java | 4 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 2 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 101 ++++-
.../key/TestOMKeyPurgeRequestAndResponse.java | 2 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 2 +-
.../TestS3MultipartUploadCompleteRequest.java | 2 +-
.../request/volume/TestOMVolumeCreateRequest.java | 2 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 2 +-
.../ozone/om/response/TestCleanupTableInfo.java | 4 +-
.../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 2 +-
.../om/upgrade/TestOzoneManagerPrepareState.java | 2 +-
.../ozone/security/TestAWSV4AuthValidator.java | 2 +-
.../TestOzoneDelegationTokenSecretManager.java | 6 +-
.../ozone/security/TestOzoneTokenIdentifier.java | 6 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 10 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 16 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 9 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 18 +
.../fs/ozone/BasicRootedOzoneFileSystem.java | 3 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 +
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 24 +-
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 2 +-
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 59 +++
hadoop-ozone/ozonefs-shaded/pom.xml | 2 +-
.../org/hadoop/ozone/recon/codegen/SqlDbUtils.java | 4 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 2 +
.../org/apache/hadoop/ozone/recon/ReconServer.java | 125 ++++++
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 7 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 29 ++
.../ozone/recon/api/ClusterStateEndpoint.java | 15 +-
.../ozone/recon/api/MetricsProxyEndpoint.java | 4 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 2 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 2 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 4 +-
.../recon/metrics/OzoneManagerSyncMetrics.java | 12 +
.../recon/metrics/ReconTaskStatusMetrics.java | 83 ++++
.../ozone/recon/scm/ReconContainerManager.java | 4 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 18 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 2 +-
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 61 ++-
.../scm/ReconStorageContainerManagerFacade.java | 11 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 2 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 68 ++-
.../impl/StorageContainerServiceProviderImpl.java | 16 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 4 +-
.../hadoop/ozone/recon/tasks/TableCountTask.java | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +
.../ozone/recon/api/TestOpenContainerCount.java | 8 +-
.../ozone/recon/api/TestTaskStatusService.java | 2 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 2 +-
.../TestUtilizationSchemaDefinition.java | 2 +-
.../scm/AbstractReconContainerManagerTest.java | 3 +-
.../ozone/recon/scm/TestReconNodeManager.java | 12 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 3 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 88 +++-
.../impl/TestReconNamespaceSummaryManagerImpl.java | 6 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 2 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 4 +-
.../ozone/recon/tasks/TestTableCountTask.java | 2 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 19 +-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 8 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 58 +--
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 24 +-
.../ozone/s3/endpoint/ListBucketResponse.java | 6 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 108 +++--
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 6 +-
.../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 4 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 20 +-
.../hadoop/ozone/s3/signature/Credential.java | 2 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 2 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 39 +-
.../ozone/s3/TestVirtualHostStyleFilter.java | 2 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 2 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 6 +-
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 6 +-
.../admin/scm/FinalizeUpgradeCommandUtil.java | 4 +-
.../hadoop/ozone/audit/parser/AuditParser.java | 2 +-
.../ozone/audit/parser/common/DatabaseHelper.java | 16 +-
.../parser/handler/TemplateCommandHandler.java | 2 +-
.../ozone/audit/parser/model/AuditEntry.java | 22 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 6 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 8 +-
.../apache/hadoop/ozone/debug/PrefixParser.java | 2 +-
.../apache/hadoop/ozone/debug/ReadReplicas.java | 19 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 2 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 12 +-
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 4 +-
.../hadoop/ozone/freon/HadoopFsGenerator.java | 2 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 2 +-
.../GenerateOzoneRequiredConfigurations.java | 2 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 4 +-
.../ozone/shell/bucket/CreateBucketHandler.java | 2 +-
.../hadoop/ozone/audit/parser/TestAuditParser.java | 2 +-
.../hadoop/ozone/conf/TestGetConfOptions.java | 4 +-
.../apache/hadoop/ozone/freon/TestProgressBar.java | 6 +-
.../TestGenerateOzoneRequiredConfigurations.java | 4 +-
.../org/apache/hadoop/test/OzoneTestDriver.java | 6 +-
pom.xml | 30 +-
719 files changed, 6653 insertions(+), 3426 deletions(-)
diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java
index 6f563eb,dbf634f..c32965f
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java
@@@ -19,14 -17,10 +17,13 @@@
package org.apache.hadoop.hdds.security.x509.certificate.client;
- import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
++import java.io.IOException;
+
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+ import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+
- import java.io.IOException;
+ import org.slf4j.Logger;
import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.FAILURE;
import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.GETCERT;
@@@ -34,36 -28,38 +31,38 @@@ import static org.apache.hadoop.hdds.se
import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.SUCCESS;
/**
- * Certificate client for OzoneManager.
+ * Common Certificate client.
*/
- public class OMCertificateClient extends DefaultCertificateClient {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(OMCertificateClient.class);
+ public class CommonCertificateClient extends DefaultCertificateClient {
- public static final String COMPONENT_NAME = "om";
-
- public OMCertificateClient(SecurityConfig securityConfig,
- String certSerialId, String localCrlId) throws IOException {
- super(securityConfig, LOG, certSerialId, COMPONENT_NAME);
- this.setLocalCrlId(localCrlId!=null ?
- Long.parseLong(localCrlId): 0);
- }
+ private final Logger log;
- public OMCertificateClient(SecurityConfig securityConfig,
- String certSerialId) throws IOException{
- this(securityConfig, certSerialId, null);
+ public CommonCertificateClient(SecurityConfig securityConfig, Logger log,
- String certSerialId, String component) {
++ String certSerialId, String component) throws IOException {
+ super(securityConfig, log, certSerialId, component);
+ this.log = log;
}
- public OMCertificateClient(SecurityConfig securityConfig) throws IOException {
- this(securityConfig, null, null);
+ /**
+ * Returns a CSR builder that can be used to creates a Certificate signing
+ * request.
+ *
+ * @return CertificateSignRequest.Builder
+ */
+ @Override
+ public CertificateSignRequest.Builder getCSRBuilder()
+ throws CertificateException {
+ return super.getCSRBuilder()
+ .setDigitalEncryption(true)
+ .setDigitalSignature(true);
}
@Override
- protected InitResponse handleCase(InitCase init) throws IOException {
+ protected InitResponse handleCase(InitCase init)
- throws CertificateException {
++ throws CertificateException, IOException {
switch (init) {
case NONE:
- LOG.info("Creating keypair for client as keypair and certificate not " +
+ log.info("Creating keypair for client as keypair and certificate not " +
"found.");
bootstrapClientKeys();
return GETCERT;
diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
index 8f88a38,d681806..f5b2405
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
@@@ -789,9 -686,9 +789,9 @@@ public abstract class DefaultCertificat
*
*/
@Override
- public synchronized InitResponse init() throws CertificateException {
+ public synchronized InitResponse init() throws IOException {
int initCase = 0;
- PrivateKey pvtKey= getPrivateKey();
+ PrivateKey pvtKey = getPrivateKey();
PublicKey pubKey = getPublicKey();
X509Certificate certificate = getCertificate();
diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java
index 6f563eb,d6c535a..3ce7d16
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java
@@@ -24,15 -23,7 +23,9 @@@ import org.slf4j.Logger
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
- import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import java.io.IOException;
+
- import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.FAILURE;
- import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.GETCERT;
- import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.RECOVER;
- import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.SUCCESS;
-
/**
* Certificate client for OzoneManager.
*/
@@@ -44,14 -35,14 +37,14 @@@ public class OMCertificateClient extend
public static final String COMPONENT_NAME = "om";
public OMCertificateClient(SecurityConfig securityConfig,
- String certSerialId, String localCrlId) {
+ String certSerialId, String localCrlId) throws IOException {
super(securityConfig, LOG, certSerialId, COMPONENT_NAME);
- this.setLocalCrlId(localCrlId!=null ?
- Long.parseLong(localCrlId): 0);
+ this.setLocalCrlId(localCrlId != null ?
+ Long.parseLong(localCrlId) : 0);
}
public OMCertificateClient(SecurityConfig securityConfig,
- String certSerialId) throws IOException{
- String certSerialId) {
++ String certSerialId) throws IOException {
this(securityConfig, certSerialId, null);
}
diff --cc hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java
index 12490c5,afbcbf6..482d81a
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java
@@@ -15,35 -15,28 +15,31 @@@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
- package org.apache.hadoop.hdds.scm.cli;
+ package org.apache.hadoop.hdds.security.x509.certificate.client;
+import java.io.IOException;
+
- import org.apache.hadoop.hdds.cli.HddsVersionProvider;
- import org.apache.hadoop.hdds.scm.client.ScmClient;
+ import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import picocli.CommandLine.Command;
/**
- * This is the handler that process safe mode exit command.
+ * Certificate client for Recon.
*/
- @Command(
- name = "exit",
- description = "Force SCM out of safe mode",
- mixinStandardHelpOptions = true,
- versionProvider = HddsVersionProvider.class)
- public class SafeModeExitSubcommand extends ScmSubcommand {
-
+ public class ReconCertificateClient extends CommonCertificateClient {
private static final Logger LOG =
- LoggerFactory.getLogger(SafeModeExitSubcommand.class);
+ LoggerFactory.getLogger(ReconCertificateClient.class);
+
+ public static final String COMPONENT_NAME = "recon";
+
+ public ReconCertificateClient(SecurityConfig securityConfig,
- String certSerialId) {
++ String certSerialId) throws IOException {
+ super(securityConfig, LOG, certSerialId, COMPONENT_NAME);
+ }
@Override
- public void execute(ScmClient scmClient) throws IOException {
- boolean execReturn = scmClient.forceExitSafeMode();
- if(execReturn){
- LOG.info("SCM exit safe mode successfully.");
- }
+ public Logger getLogger() {
+ return LOG;
}
}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index ede522d,a4171e8..60f1099
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@@ -301,13 -299,7 +315,13 @@@ public class SCMSecurityProtocolServer
*/
@Override
public String getCACertificate() throws IOException {
- LOGGER.debug("Getting CA certificate.");
+ if (securityConfig.isCustomCAEnabled()) {
+ throw new SCMSecurityException("Get CA Certificate is not supported " +
+ "when custom CA is enabled.");
+ }
- if(LOGGER.isDebugEnabled()) {
++ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Getting CA certificate.");
+ }
try {
return CertificateCodec.getPEMEncodedString(
scmCertificateServer.getCACertificate());
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 8d8abdd,05ac12c..7eb5a3d
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -809,10 -817,10 +833,10 @@@ public final class StorageContainerMana
* @param configurator - configurator
* @throws IOException - on Failure
*/
- private void initalizeMetadataStore(OzoneConfiguration conf,
- SCMConfigurator configurator)
+ private void initializeMetadataStore(OzoneConfiguration conf,
+ SCMConfigurator configurator)
throws IOException {
- if(configurator.getMetadataStore() != null) {
+ if (configurator.getMetadataStore() != null) {
scmMetadataStore = configurator.getMetadataStore();
} else {
scmMetadataStore = new SCMMetadataStoreImpl(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org