You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by as...@apache.org on 2025/05/16 16:12:10 UTC
(ozone) 03/04: Merge branch 'master' of https://github.com/apache/ozone into HDDS-10239-container-reconciliation
This is an automated email from the ASF dual-hosted git repository.
aswinshakil pushed a commit to branch HDDS-10239-container-reconciliation
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 2b4708b70e5c9de133381a38ca7fa4b3cf3caa42
Merge: 7c591b4198 8ceb5c3197
Author: Aswin Shakil Balasubramanian <as...@gmail.com>
AuthorDate: Fri May 16 12:54:29 2025 +0530
Merge branch 'master' of https://github.com/apache/ozone into HDDS-10239-container-reconciliation
Commits: 113 commits
8ceb5c3197 Revert "HDDS-11232. Spare InfoBucket RPC call for the FileSystem#getFileStatus calls for the general case. (#6988)" (#8358)
8ecf4a83d8 HDDS-12993. Use DatanodeID in ReconNodeManager. (#8456)
3e09a11ed8 HDDS-13024. HTTP connect to 0.0.0.0 failed (#8439)
c3cb14f317 HDDS-12914. Bump awssdk to 2.31.40, test ResumableFileDownload (#8455)
c6b47e90e0 HDDS-4677. Document Ozone Ports and Connection End Points (#8226)
ae0d757f0d HDDS-13044. Remove DatanodeDetails#getUuid usages from hdds-common/client/container-service (#8462)
2bf3f16c38 HDDS-12568. Implement MiniOzoneCluster.Service for Recon (#8452)
e59d251a52 HDDS-13046. Add .vscode to .gitignore (#8461)
50e6d616a3 HDDS-13022. Split up exclusive size tracking for key and directory cleanup in SnapshotInfo (#8433)
6db647d9f2 HDDS-12966. DBDefinitionFactory should not throw InvalidArnException (#8454)
fe2875c119 HDDS-12948. [Snapshot] Increase `ozone.om.fs.snapshot.max.limit` default value to 10k (#8376)
215bdc272d HDDS-11904. Fix HealthyPipelineSafeModeRule logic. (#8386)
dba75575fd HDDS-12989. Throw CodecException for the Codec byte[] methods (#8444)
d9c73f0000 HDDS-11298. Support owner field in the S3 listBuckets request (#8315)
7a5129d695 HDDS-12810. Check and reserve space atomically in VolumeChoosingPolicy (#8360)
d51ccf72d8 HDDS-13016. Add a getAllNodeCount() method to NodeManager. (#8445)
8b27fb92b8 HDDS-12299. Merge OzoneAclConfig into OmConfig (#8383)
96f4f79503 HDDS-12501. Remove OMKeyInfo from SST files in backup directory. (#8214)
8adb50002a HDDS-13021. Make callId unique for each request in AbstractKeyDeletingService (#8432)
faeadd6b14 HDDS-13000. [Snapshot] listSnapshotDiffJobs throws NPE. (#8418)
7c957cce1c HDDS-11954. List keys command gets wrong info from BasicOmKeyInfo (#8401)
0697383d35 HDDS-13027. Workaround for GitHub connection failures (#8443)
0d5f933819 HDDS-12919. Remove redundant FileLock from ChunkWrite (#8435)
395892a6a9 HDDS-13012. [Snapshot] Add audit logs for listSnapshotDiffJobs, snapshotDiff and cancelSnapshotDiff. (#8425)
d92f0ed87e HDDS-12653. Add option in `ozone debug log container list` to filter by Health State (#8415)
78f8a0240b HDDS-12645. Improve output of `ozone debug replicas chunk-info` (#8146)
f32470cebf HDDS-12995. Split ListOptions for better reuse (#8413)
cdd1b132a0 HDDS-13019. Bump jline to 3.30.0 (#8428)
1651408fe4 HDDS-13018. Bump common-custom-user-data-maven-extension to 2.0.2 (#8429)
8f6222e756 HDDS-12949. Cleanup SCMSafeModeManager (#8390)
adb385f02a HDDS-13002. Use DatanodeID in ContainerBalancerTask (#8421)
2b2a7965f4 HDDS-12969. Use DatanodeID in XceiverClientGrpc (#8430)
3c0bafae58 HDDS-13001. Use DatanodeID in SCMBlockDeletingService. (#8420)
8448bc2c9c HDDS-12187. Use hadoop-dependency-client as parent, not dependency (#8416)
582fd0307f HDDS-13015. Bump zstd-jni to 1.5.7-3 (#8426)
ec12f98eac HDDS-12970. Use DatanodeID in Pipeline (#8414)
3f48a18c51 HDDS-12179. Improve Container Log to also capture the Volume Information (#7858)
034b4973fc HDDS-12964. Add minimum required version of Docker Compose to RunningViaDocker page (#8385)
c0a6b42451 HDDS-12947. Add CodecException (#8411)
70949c5298 HDDS-11603. Reclaimable Key Filter for Snapshots garbage reclaimation (#8392)
08283f3aca HDDS-12087. TransactionToDNCommitMap too large causes GC to pause for a long time (#8347)
f47df78a5d HDDS-12958. [Snapshot] Add ACL check regression tests for snapshot operations. (#8419)
1cc8445847 HDDS-12207. Unify output of ozone debug replicas verify checks (#8248)
f087d0b8e4 HDDS-12994. Use DatanodeID in ReconSCMDBDefinition. (#8417)
d6a7723c07 HDDS-12776. ozone debug CLI command to list all Duplicate open containers (#8409)
c78aeb0ed3 HDDS-12951. EC: Log when falling back to reconstruction read (#8408)
412f22d1b7 HDDS-12959. Eliminate hdds-hadoop-dependency-server (#8384)
df701dc25d HDDS-12996. Workaround for Docker Compose concurrent map writes (#8412)
e6daae4af3 HDDS-12972. Use DatanodeID in ContainerReplica. (#8396)
c1103aeb27 HDDS-12877. Support StorageClass field in the S3 HeadObject request (#8351)
4135384801 HDDS-12973. Add javadoc for CompactionNode() and make getCompactionNodeGraph return ConcurrentMap (#8395)
4f2e13cad5 HDDS-12954. Do not throw IOException for checksum. (#8387)
49b8fbd890 HDDS-12971. Use DatanodeID in Node2PipelineMap (#8403)
d2da18f69b HDDS-12346. Reduce code duplication among TestNSSummaryTask classes (#8287)
82b73e3db4 HDDS-11856. Set DN state machine thread priority higher than command handler thread. (#8253)
d4f2734aaa HDDS-12689. Import BOM for AWS SDK, declare dependencies (#8406)
4775e76cfb HDDS-12975. Fix percentage of blocks deleted in grafana dashboard (#8398)
ac0d696608 HDDS-12968. [Recon] Fix column visibility issue in Derby during schema upgrade finalization. (#8393)
e71dcf6eb4 HDDS-11981. Add annotation for registering feature validator based on a generic version (#7603)
254297c726 HDDS-12562. Reclaimable Directory entry filter for reclaiming deleted directory entries (#8055)
4f467c8fc7 HDDS-12978. Remove TestMultipartObjectGet (#8400)
a99f20710b HDDS-12967. Skip CommonChunkManagerTestCases.testFinishWrite if fuser cannot be started (#8389)
d29d76ba90 HDDS-12697. Ozone debug CLI to display details of a single container (#8264)
1d1bc883b7 HDDS-12974. Docker could not parse extra host IP (#8397)
7e675d743a HDDS-12053. Make print-log-dag command run locally and offline (#8016)
d3faab30c5 HDDS-12561. Reclaimable Rename entry filter for reclaiming renaming entries (#8054)
af1f98cb37 HDDS-10822. Tool to omit raft log in OM. (#8154)
3201ca4e32 HDDS-12952. Make OmSnapshotManager#snapshotLimitCheck thread-safe and consistent (#8381)
522c88d364 HDDS-12963. Clean up io.grpc dependencies (#8382)
fa8bd9dd70 HDDS-12916. Support ETag in listObjects response (#8356)
fdc77db35e HDDS-12300. Merge OmUpgradeConfig into OmConfig (#8378)
623e1446b2 HDDS-12956. Bump vite to 4.5.14 (#8375)
8c8eaf1377 HDDS-12944. Reduce timeout for integration check (#8374)
40d2e00f54 HDDS-11141. Avoid log flood due due pipeline close in XceiverServerRatis (#8325)
9fe1dba2aa HDDS-12942. Init layout version config should not be public (#8373)
452e7aa789 HDDS-12596. OM fs snapshot max limit is not enforced (#8377)
8b095d5d24 HDDS-12795. Rename heartbeat and first election configuration name (#8249)
bee81649c4 HDDS-12920. Configure log4j to gzip rolled over service log files (#8357)
e16a50f946 HDDS-12934. Split submodule for Freon. (#8367)
b1e95119c9 HDDS-12925. Update datanode volume used space on container deletion (#8364)
440bc82761 Revert "HDDS-12596. OM fs snapshot max limit is not enforced (#8157)"
f345492db0 HDDS-12596. OM fs snapshot max limit is not enforced (#8157)
ee7b1dc607 HDDS-12901. Introduce EventExecutorMetrics instead of setting the metrics props unsafely (#8371)
810e148ea9 HDDS-12939. Remove UnknownPipelineStateException. (#8372)
560fcdf90c HDDS-12728. Add Ozone 2.0.0 to compatibility acceptance tests (#8361)
5815a47625 HDDS-12933. Remove the table names declared in OmMetadataManagerImpl (#8370)
ee32fa5494 HDDS-12560. Reclaimable Filter for Snaphost Garbage Collections (#8053)
45374ea5bb HDDS-12932. Rewrite OMDBDefinition (#8362)
5cb6dd84d8 HDDS-12575. Set default JUnit5 timeout via property (#8348)
8efc0cd65a HDDS-11633. Delete message body too large, causing SCM to fail writing raft log (#8354)
ac9d9fdb00 HDDS-12915. Intermittent failure in testCreatePipelineThrowErrorWithDataNodeLimit (#8359)
86039e8302 HDDS-12848. Create new submodule for ozone admin (#8292)
2d0f8cb6af HDDS-12833. Remove the CodecRegistry field from DBStoreBuilder (#8327)
c71b393bef HDDS-12921. UnusedPrivateField violations in tests (#8353)
9f3dd01111 HDDS-12917. cp: option '--update' doesn't allow an argument (#8346)
a14b395cbc HDDS-12922. Use OMDBDefinition in GeneratorOm and FSORepairTool (#8355)
c8a98d6e29 HDDS-12892. OM Tagging Request incorrectly sets full path as key name for FSO (#8345)
ade69e3f82 HDDS-12649. Include name of volume or bucket in length validation error (#8322)
c68308da9e HDDS-12599. Create an ozone debug CLI command to list all the containers based on final state (#8282)
319d5a4cfe HDDS-12773. bad substitution in bats test (#8290)
6f5e02a72b HDDS-12900. (addendum: fix pmd) Use OMDBDefinition in OmMetadataManagerImpl (#8337)
2a7000dc12 HDDS-12900. Use OMDBDefinition in OmMetadataManagerImpl (#8337)
9c0c66cb69 HDDS-12915. Mark testCreatePipelineThrowErrorWithDataNodeLimit as flaky
a73e0529ba HDDS-12907. Enable FieldDeclarationsShouldBeAtStartOfClass PMD rule (#8344)
d083f8272d HDDS-12905. Move field declarations to start of class in ozone-common (#8342)
3f90e1c1b5 HDDS-12906. Move field declarations to start of class in ozone-manager module (#8343)
403fb97d96 HDDS-12878. Move field declarations to start of class in tests (#8308)
63d5c73677 HDDS-12912. Remove deprecated `PipelineManager#closePipeline(Pipeline, boolean)` (#8340)
cf1fb88740 HDDS-12902. Shutdown executor in CloseContainerCommandHandler and ECReconstructionCoordinator (#8341)
825ba02e70 HDDS-9585. Improve import/export log in ContainerLogger (#8330)
b70d35ab64 HDDS-12889. Enable AppendCharacterWithChar PMD rule (#8324)
cd308eaa85 HDDS-12904. Move field declarations to start of class in other hdds modules (#8336)
490528646b HDDS-12899. Move field declarations to start of class in hdds-server-scm (#8332)
Conflicts:
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReconcileSubcommand.java
hadoop-ozone/recon-codegen/src/main/java/org/apache/ozone/recon/schema/ContainerSchemaDefinition.java
.github/workflows/ci.yml | 3 +-
.gitignore | 1 +
.mvn/extensions.xml | 2 +-
dev-support/pmd/pmd-ruleset.xml | 8 +
...ava => OmRequestFeatureValidatorProcessor.java} | 50 +-
.../annotations/RegisterValidatorProcessor.java | 146 ++++
hadoop-hdds/client/pom.xml | 7 +-
.../hadoop/hdds/scm/ContainerClientMetrics.java | 6 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 35 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 6 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 5 +-
.../ozone/client/io/ECBlockInputStreamProxy.java | 3 +
.../hdds/scm/TestContainerClientMetrics.java | 8 +-
.../hdds/scm/client/TestHddsClientUtils.java | 103 ++-
.../storage/TestBlockOutputStreamCorrectness.java | 3 +-
hadoop-hdds/common/pom.xml | 21 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 6 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +-
.../java/org/apache/hadoop/hdds/NodeDetails.java | 2 +-
.../java/org/apache/hadoop/hdds/StringUtils.java | 25 +-
.../apache/hadoop/hdds/cli/AdminSubcommand.java | 0
.../apache/hadoop/hdds/cli/DebugSubcommand.java | 0
.../org/apache/hadoop/hdds/cli/ItemsFromStdin.java | 0
.../apache/hadoop/hdds/cli/RepairSubcommand.java | 0
.../hadoop/hdds/conf/OzoneConfiguration.java | 2 +
.../apache/hadoop/hdds/protocol/DatanodeID.java | 13 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 4 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 22 +-
.../common/helpers/ContainerWithPipeline.java | 18 +-
.../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 2 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 6 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 120 ++--
.../hadoop/hdds/scm/pipeline/PipelineID.java | 14 +-
.../pipeline/UnknownPipelineStateException.java | 45 --
.../x509/certificate/utils/CertificateCodec.java | 48 +-
.../apache/hadoop/hdds/tracing/StringCodec.java | 6 +-
.../org/apache/hadoop/hdds/utils/db/Codec.java | 49 +-
.../apache/hadoop/hdds/utils/db/CodecBuffer.java | 8 +-
.../hadoop/hdds/utils/db/CodecException.java} | 20 +-
.../hadoop/hdds/utils/db/DelegatedCodec.java | 33 +-
.../apache/hadoop/hdds/utils/db/Proto2Codec.java | 28 +-
.../apache/hadoop/hdds/utils/db/Proto3Codec.java | 28 +-
.../hadoop/hdds/utils/db/StringCodecBase.java | 10 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 13 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 7 +
.../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 3 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 51 +-
.../apache/hadoop/ozone/common/ChecksumData.java | 137 ++--
.../ozone/common/OzoneChecksumException.java | 24 +-
.../ozone/container/common/helpers/BlockData.java | 49 +-
.../hadoop/ozone/utils/FormattingCLIUtils.java | 20 +-
.../apache/hadoop/ozone/utils}/package-info.java | 4 +-
.../common/src/main/resources/ozone-default.xml | 36 +-
.../hdds/scm/net/TestNetworkTopologyImpl.java | 2 -
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 2 -
.../hadoop/hdds/scm/net/TestNodeSchemaManager.java | 2 -
.../hadoop/hdds/scm/net/TestYamlSchemaLoader.java | 2 -
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 4 +-
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 2 +-
.../hadoop/hdds/utils/TestResourceCache.java | 4 -
.../hadoop/hdds/utils/db/Proto2CodecTestBase.java | 11 +-
.../apache/hadoop/ozone/common/TestChecksum.java | 5 +-
.../hadoop/ozone/common/TestChunkBuffer.java | 4 -
hadoop-hdds/container-service/pom.xml | 32 +
.../helpers/BlockDeletingServiceMetrics.java | 30 +-
.../helpers/DeletedContainerBlocksSummary.java | 6 +-
.../common/impl/BlockDeletingService.java | 6 +-
.../ozone/container/common/impl/ContainerData.java | 20 +-
.../ozone/container/common/impl/ContainerSet.java | 20 +-
.../common/impl/StorageLocationReport.java | 2 +-
.../common/statemachine/DatanodeStateMachine.java | 10 +
.../common/statemachine/StateContext.java | 10 +-
.../CloseContainerCommandHandler.java | 5 +
.../CreatePipelineCommandHandler.java | 3 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 3 +-
.../states/endpoint/RegisterEndpointTask.java | 14 +-
.../common/transport/server/XceiverServerGrpc.java | 16 +-
.../transport/server/ratis/XceiverServerRatis.java | 9 +-
.../container/common/utils/ContainerLogger.java | 5 +-
.../volume/CapacityVolumeChoosingPolicy.java | 20 +-
.../volume/RoundRobinVolumeChoosingPolicy.java | 11 +-
.../reconstruction/ECContainerOperationClient.java | 7 +-
.../ECReconstructionCoordinator.java | 3 +-
.../container/keyvalue/KeyValueContainer.java | 13 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 9 +-
.../container/keyvalue/helpers/ChunkUtils.java | 6 +-
.../ozone/container/metadata/AbstractRDBStore.java | 2 +-
.../ozone/container/metadata/DatanodeTable.java | 2 +-
.../metadata/SchemaOneChunkInfoListCodec.java | 6 +-
.../container/metadata/SchemaOneKeyCodec.java | 4 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 100 +--
.../replication/AbstractReplicationTask.java | 2 +-
.../container/replication/ContainerImporter.java | 13 +-
.../replication/DownloadAndImportReplicator.java | 18 +-
.../replication/SendContainerRequestHandler.java | 16 +-
.../protocol/commands/CloseContainerCommand.java | 2 +-
.../protocol/commands/ClosePipelineCommand.java | 2 +-
.../protocol/commands/CommandForDatanode.java | 20 +-
.../protocol/commands/CreatePipelineCommand.java | 2 +-
.../protocol/commands/DeleteBlocksCommand.java | 4 +-
.../protocol/commands/DeleteContainerCommand.java | 2 +-
.../commands/FinalizeNewLayoutVersionCommand.java | 2 +-
.../commands/ReconstructECContainersCommand.java | 4 +-
.../commands/RefreshVolumeUsageCommand.java | 2 +-
.../commands/ReplicateContainerCommand.java | 2 +-
.../ozone/protocol/commands/ReregisterCommand.java | 2 +-
.../hadoop/ozone/protocol/commands/SCMCommand.java | 5 +
.../commands/SetNodeOperationalStateCommand.java | 2 +-
.../container/common/TestBlockDeletingService.java | 2 -
.../container/common/TestDatanodeStateMachine.java | 20 +
.../container/common/helpers/TestBlockData.java | 2 -
.../common/helpers/TestDatanodeIdYaml.java | 1 -
.../common/impl/TestContainerPersistence.java | 7 +-
.../container/common/interfaces/TestHandler.java | 2 -
.../TestDeleteBlocksCommandHandler.java | 2 -
.../container/common/utils/TestHddsVolumeUtil.java | 4 +-
.../volume/TestCapacityVolumeChoosingPolicy.java | 11 +
.../container/common/volume/TestDbVolume.java | 2 +-
.../common/volume/TestPeriodicVolumeChecker.java | 2 -
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 13 +
.../common/volume/TestStorageVolumeChecker.java | 52 +-
.../container/common/volume/TestVolumeSet.java | 2 -
.../common/volume/TestVolumeSetDiskChecks.java | 6 -
.../container/keyvalue/TestKeyValueContainer.java | 17 +
.../TestKeyValueContainerMarkUnhealthy.java | 6 -
.../container/keyvalue/TestKeyValueHandler.java | 136 +++-
.../keyvalue/impl/AbstractTestChunkManager.java | 8 +-
.../container/ozoneimpl/TestContainerReader.java | 33 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
... ReplicationSupervisorSchedulingBenchmark.java} | 19 +-
.../TestDownloadAndImportReplicator.java | 123 ++++
.../container/replication/TestPushReplicator.java | 2 -
.../replication/TestReplicationSupervisor.java | 31 +-
.../TestSendContainerRequestHandler.java | 124 +++-
.../replication/TestSimpleContainerDownloader.java | 2 -
hadoop-hdds/docs/content/concept/NetworkPorts.md | 102 +++
hadoop-hdds/docs/content/start/RunningViaDocker.md | 3 +-
hadoop-hdds/erasurecode/pom.xml | 7 +-
.../ozone/erasurecode/rawcoder/CoderUtil.java | 6 +-
.../ozone/erasurecode/rawcoder/util/GF256.java | 5 +-
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 7 +-
.../apache/ozone/erasurecode/TestCoderBase.java | 4 +-
.../rawcoder/RawErasureCoderBenchmark.java | 10 +-
hadoop-hdds/framework/pom.xml | 61 +-
...inerLocationProtocolClientSideTranslatorPB.java | 14 +-
.../hdds/security/x509/certificate/CertInfo.java | 34 +-
.../hdds/server/events/EventExecutorMetrics.java | 143 ++++
.../FixedThreadPoolWithAffinityExecutor.java | 60 +-
.../hdds/server/events/SingleThreadExecutor.java | 39 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 4 +-
.../hadoop/hdds/server/http/HttpServer2.java | 2 +-
.../hdds/server/http/PrometheusMetricsSink.java | 6 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 28 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 2 +-
.../org/apache/hadoop/hdds/utils/MetricsUtil.java | 100 ---
.../org/apache/hadoop/hdds/utils/SignalLogger.java | 2 +-
.../hdds/utils/db/DBColumnFamilyDefinition.java | 28 +-
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 33 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 58 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 20 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 3 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 28 +-
.../hdds/server/http/TestBaseHttpServer.java | 130 +++-
.../hadoop/hdds/utils/db/InMemoryTestTable.java | 12 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 17 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 23 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 13 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 15 +-
.../src/test/resources/webapps/testing/.gitkeep | 15 +
hadoop-hdds/hadoop-dependency-client/pom.xml | 370 +++++-----
hadoop-hdds/hadoop-dependency-server/pom.xml | 210 ------
.../interface-client/src/main/proto/hdds.proto | 1 +
.../db/managed/ManagedRocksObjectMetrics.java | 12 +-
.../utils/db/managed/ManagedRocksObjectUtils.java | 5 +-
hadoop-hdds/pom.xml | 132 ----
.../apache/hadoop/hdds/utils/NativeConstants.java | 6 +-
.../utils/db/managed/ManagedRawSSTFileReader.java | 11 +-
.../ozone/compaction/log/CompactionFileInfo.java | 34 +-
.../ozone/compaction/log/CompactionLogEntry.java | 25 +-
.../org/apache/ozone/rocksdiff/CompactionDag.java | 154 ++++
.../org/apache/ozone/rocksdiff/CompactionNode.java | 9 +-
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 455 ++++++------
.../compaction/log/TestCompactionFileInfo.java | 29 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 500 ++++++++-----
.../apache/ozone/rocksdiff/TestRocksDiffUtils.java | 4 +-
hadoop-hdds/server-scm/pom.xml | 21 +-
.../block/DatanodeDeletedBlockTransactions.java | 10 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 8 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 56 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 14 +-
.../SCMDeletedBlockTransactionStatusManager.java | 4 +
.../scm/block/ScmBlockDeletingServiceMetrics.java | 28 +-
.../container/AbstractContainerReportHandler.java | 4 +-
.../scm/container/CloseContainerEventHandler.java | 2 +-
.../hdds/scm/container/ContainerReplica.java | 74 +-
.../hdds/scm/container/ContainerReportHandler.java | 16 +-
.../container/balancer/ContainerBalancerTask.java | 18 +-
.../ContainerBalancerTaskIterationStatusInfo.java | 8 +-
.../hdds/scm/container/balancer/DataMoveInfo.java | 14 +-
.../hdds/scm/container/balancer/MoveManager.java | 98 +--
.../ReconcileContainerEventHandler.java | 2 +-
.../replication/ContainerHealthResult.java | 24 +-
.../container/replication/ContainerReplicaOp.java | 13 +-
.../replication/ECContainerReplicaCount.java | 2 +-
.../replication/QuasiClosedStuckReplicaCount.java | 67 +-
.../replication/RatisContainerReplicaCount.java | 4 +-
.../container/replication/ReplicationManager.java | 64 +-
.../replication/ReplicationManagerUtil.java | 11 +-
.../scm/container/states/ContainerStateMap.java | 38 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 5 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 16 +-
.../apache/hadoop/hdds/scm/ha/SCMHAMetrics.java | 59 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 14 +-
.../hadoop/hdds/scm/metadata/BigIntegerCodec.java | 5 +-
.../hdds/scm/metadata/X509CertificateCodec.java | 47 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 18 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 2 +-
.../hdds/scm/node/NodeDecommissionManager.java | 4 +-
.../hdds/scm/node/NodeDecommissionMetrics.java | 152 ++--
.../apache/hadoop/hdds/scm/node/NodeManager.java | 11 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 23 +-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 42 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 49 +-
.../hdds/scm/node/states/Node2PipelineMap.java | 16 +-
.../hdds/scm/pipeline/ECPipelineProvider.java | 18 +-
.../hdds/scm/pipeline/PipelineActionHandler.java | 2 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 4 -
.../hdds/scm/pipeline/PipelineManagerImpl.java | 14 -
.../hdds/scm/pipeline/PipelineReportHandler.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 6 +-
.../scm/pipeline/WritableECContainerProvider.java | 20 +-
.../algorithms/MinLeaderCountChoosePolicy.java | 2 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 75 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 300 +++-----
.../hadoop/hdds/scm/safemode/SafeModeExitRule.java | 4 +-
.../hadoop/hdds/scm/safemode/SafeModeMetrics.java | 14 +-
.../hdds/scm/safemode/SafeModeRuleFactory.java | 10 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 4 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 6 +-
.../hadoop/hdds/scm/server/SCMPolicyProvider.java | 18 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 5 +-
.../hdds/scm/server/StorageContainerManager.java | 12 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 2 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 26 +-
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 2 -
.../hadoop/hdds/scm/TestHddsServerUtils.java | 6 -
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 10 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 20 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 59 +-
.../scm/block/TestSCMBlockDeletingService.java | 6 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 4 +-
.../container/TestCloseContainerEventHandler.java | 2 +-
.../hdds/scm/container/TestContainerReplica.java | 4 +-
.../container/balancer/TestContainerBalancer.java | 2 -
.../balancer/TestContainerBalancerStatusInfo.java | 4 +-
.../balancer/TestContainerBalancerTask.java | 2 +-
.../scm/container/balancer/TestableCluster.java | 2 +-
.../container/replication/ReplicationTestUtil.java | 20 +-
...TestQuasiClosedStuckOverReplicationHandler.java | 6 +-
.../TestQuasiClosedStuckReplicaCount.java | 20 +-
...estQuasiClosedStuckUnderReplicationHandler.java | 18 +-
.../TestRatisContainerReplicaCount.java | 6 +-
.../replication/TestReplicationManager.java | 5 +-
.../TestReplicationManagerScenarios.java | 11 +-
.../health/TestEmptyContainerHandler.java | 2 +-
.../health/TestQuasiClosedContainerHandler.java | 12 +-
.../TestQuasiClosedStuckReplicationCheck.java | 9 +-
.../health/TestRatisReplicationCheckHandler.java | 16 +-
.../container/states/TestContainerAttribute.java | 3 +-
.../hdds/scm/ha/TestBackgroundSCMService.java | 2 +-
.../scm/ha/TestInterSCMGrpcProtocolService.java | 9 +-
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 4 +-
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 8 +-
.../scm/metadata/OldPipelineIDCodecForTesting.java | 17 +-
.../OldX509CertificateCodecForTesting.java | 21 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 4 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 4 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 16 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 9 +-
.../scm/pipeline/MockRatisPipelineProvider.java | 2 +-
.../hdds/scm/pipeline/TestECPipelineProvider.java | 4 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 25 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 36 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 20 +-
.../scm/safemode/TestDataNodeSafeModeRule.java | 10 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 111 ++-
.../TestOneReplicaPipelineSafeModeRule.java | 10 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 106 ++-
.../org/apache/ozone/test/LambdaTestUtils.java | 8 +-
.../apache/ozone/test/TimedOutTestsListener.java | 2 +-
.../tools => hadoop-ozone/cli-admin}/pom.xml | 70 +-
.../hdds/scm/cli/ContainerBalancerCommands.java | 0
.../scm/cli/ContainerBalancerStartSubcommand.java | 0
.../scm/cli/ContainerBalancerStatusSubcommand.java | 0
.../scm/cli/ContainerBalancerStopSubcommand.java | 0
.../hdds/scm/cli/ContainerOperationClient.java | 0
.../hdds/scm/cli/ReplicationManagerCommands.java | 0
.../scm/cli/ReplicationManagerStartSubcommand.java | 0
.../cli/ReplicationManagerStatusSubcommand.java | 0
.../scm/cli/ReplicationManagerStopSubcommand.java | 0
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 0
.../hadoop/hdds/scm/cli/SafeModeCommands.java | 0
.../hdds/scm/cli/SafeModeExitSubcommand.java | 0
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 0
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 0
.../apache/hadoop/hdds/scm/cli/ScmSubcommand.java | 0
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 4 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 0
.../scm/cli/cert/CleanExpiredCertsSubcommand.java | 0
.../hadoop/hdds/scm/cli/cert/InfoSubcommand.java | 0
.../hadoop/hdds/scm/cli/cert/ListSubcommand.java | 0
.../hdds/scm/cli/cert/ScmCertSubcommand.java | 0
.../hadoop/hdds/scm/cli/cert/package-info.java | 0
.../hdds/scm/cli/container/CloseSubcommand.java | 0
.../hdds/scm/cli/container/ContainerCommands.java | 0
.../scm/cli/container/ContainerIDParameters.java | 0
.../hdds/scm/cli/container/CreateSubcommand.java | 0
.../hdds/scm/cli/container/InfoSubcommand.java | 6 +-
.../hdds/scm/cli/container/ListSubcommand.java | 0
.../scm/cli/container/ReconcileSubcommand.java | 0
.../hdds/scm/cli/container/ReportSubcommand.java | 0
.../hdds/scm/cli/container/UpgradeSubcommand.java | 0
.../hdds/scm/cli/container/package-info.java | 0
.../hdds/scm/cli/datanode/DatanodeCommands.java | 0
.../cli/datanode/DecommissionStatusSubCommand.java | 4 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 0
.../hdds/scm/cli/datanode/HostNameParameters.java | 0
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 8 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 0
.../scm/cli/datanode/RecommissionSubCommand.java | 0
.../hdds/scm/cli/datanode/StatusSubCommand.java | 0
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 42 +-
.../hadoop/hdds/scm/cli/datanode/package-info.java | 0
.../apache/hadoop/hdds/scm/cli/package-info.java | 0
.../cli/pipeline/ActivatePipelineSubcommand.java | 0
.../scm/cli/pipeline/ClosePipelineSubcommand.java | 0
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 0
.../cli/pipeline/DeactivatePipelineSubcommand.java | 0
.../scm/cli/pipeline/FilterPipelineOptions.java | 0
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 0
.../hdds/scm/cli/pipeline/PipelineCommands.java | 0
.../hadoop/hdds/scm/cli/pipeline/package-info.java | 0
.../org/apache/hadoop/hdds/util/DurationUtil.java | 0
.../org/apache/hadoop/hdds/util/package-info.java | 0
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 0
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 10 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 0
.../ozone/admin/nssummary/NSSummaryAdmin.java | 0
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 8 +-
.../admin/nssummary/QuotaUsageSubCommand.java | 0
.../ozone/admin/nssummary/SummarySubCommand.java | 0
.../hadoop/ozone/admin/nssummary/package-info.java | 0
.../ozone/admin/om/CancelPrepareSubCommand.java | 0
.../ozone/admin/om/DecommissionOMSubcommand.java | 0
.../hadoop/ozone/admin/om/FetchKeySubCommand.java | 0
.../admin/om/FinalizationStatusSubCommand.java | 0
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 0
.../ozone/admin/om/GetServiceRolesSubcommand.java | 11 -
.../ozone/admin/om/ListOpenFilesSubCommand.java | 0
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 0
.../hadoop/ozone/admin/om/PrepareSubCommand.java | 0
.../ozone/admin/om/TransferOmLeaderSubCommand.java | 0
.../ozone/admin/om/UpdateRangerSubcommand.java | 0
.../ozone/admin/om/lease/LeaseRecoverer.java | 0
.../ozone/admin/om/lease/LeaseSubCommand.java | 0
.../hadoop/ozone/admin/om/lease/package-info.java | 0
.../apache/hadoop/ozone/admin/om/package-info.java | 0
.../apache/hadoop/ozone/admin/package-info.java | 0
.../reconfig/AbstractReconfigureSubCommand.java | 0
.../ozone/admin/reconfig/ReconfigureCommands.java | 0
.../reconfig/ReconfigurePropertiesSubcommand.java | 0
.../admin/reconfig/ReconfigureStartSubcommand.java | 0
.../reconfig/ReconfigureStatusSubcommand.java | 0
.../admin/reconfig/ReconfigureSubCommandUtil.java | 0
.../hadoop/ozone/admin/reconfig/package-info.java | 0
.../ozone/admin/scm/DecommissionScmSubcommand.java | 0
.../ozone/admin/scm/DeletedBlocksTxnCommands.java | 0
.../admin/scm/FinalizationScmStatusSubcommand.java | 0
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 0
.../scm/GetFailedDeletedBlocksTxnSubcommand.java | 24 +-
.../admin/scm/GetScmRatisRolesSubcommand.java | 0
.../scm/ResetDeletedBlockRetryCountSubcommand.java | 0
.../ozone/admin/scm/RotateKeySubCommand.java | 0
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 0
.../admin/scm/TransferScmLeaderSubCommand.java | 0
.../hadoop/ozone/admin/scm/package-info.java | 0
.../cli/cert/TestCleanExpiredCertsSubcommand.java | 0
.../hdds/scm/cli/container/TestInfoSubCommand.java | 0
.../scm/cli/container/TestReportSubCommand.java | 2 +-
.../datanode/TestContainerBalancerSubCommand.java | 0
.../datanode/TestDecommissionStatusSubCommand.java | 0
.../cli/datanode/TestDecommissionSubCommand.java | 0
.../scm/cli/datanode/TestListInfoSubcommand.java | 0
.../cli/datanode/TestMaintenanceSubCommand.java | 0
.../cli/datanode/TestRecommissionSubCommand.java | 0
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 0
.../cli/pipeline/TestClosePipelinesSubCommand.java | 0
.../cli/pipeline/TestListPipelinesSubCommand.java | 0
.../apache/hadoop/hdds/util/TestDurationUtil.java | 0
.../ozone/scm/TestDecommissionScmSubcommand.java | 0
.../ozone/scm/TestGetScmRatisRolesSubcommand.java | 0
.../org/apache/hadoop/ozone/scm/package-info.java | 0
hadoop-ozone/cli-shell/pom.xml | 11 +-
.../{ListOptions.java => ListLimitOptions.java} | 57 +-
.../hadoop/ozone/shell/ListPaginationOptions.java | 32 +-
.../hadoop/ozone/shell/PrefixFilterOption.java} | 23 +-
.../ozone/shell/bucket/CreateBucketHandler.java | 44 +-
.../ozone/shell/bucket/ListBucketHandler.java | 10 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 18 +-
.../shell/snapshot/ListSnapshotDiffHandler.java | 4 +-
.../ozone/shell/snapshot/ListSnapshotHandler.java | 10 +-
.../ozone/shell/volume/ListVolumeHandler.java | 12 +-
.../hadoop/ozone/shell/TestOzoneAddress.java | 4 +-
hadoop-ozone/client/pom.xml | 7 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 2 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 5 +-
.../apache/hadoop/ozone/client/OzoneSnapshot.java | 28 +-
.../checksum/ReplicatedBlockChecksumComputer.java | 4 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 16 +-
.../ozone/client/io/KeyDataStreamOutput.java | 17 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 18 +-
.../ozone/client/protocol/ClientProtocol.java | 13 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 2 +-
.../hadoop/ozone/client/TestOzoneSnapshot.java | 63 ++
hadoop-ozone/common/pom.xml | 25 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 6 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 7 +-
.../ozone/client/io/SelectorOutputStream.java | 7 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 18 +-
.../java/org/apache/hadoop/ozone/om/OmConfig.java | 76 ++
.../apache/hadoop/ozone/om/OmUpgradeConfig.java | 78 ---
.../hadoop/ozone/om/exceptions/OMException.java | 4 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 8 +-
.../hadoop/ozone/om/helpers/OmDBAccessIdInfo.java | 13 +-
.../hadoop/ozone/om/helpers/OmDBTenantState.java | 8 +-
.../ozone/om/helpers/OmDBUserPrincipalInfo.java | 8 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 8 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 37 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 52 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 8 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 62 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 8 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 17 +-
.../hadoop/ozone/om/helpers/OzoneFileStatus.java | 4 +-
.../ozone/om/helpers/OzoneFileStatusLight.java | 4 +-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 10 +-
.../hadoop/ozone/om/helpers/S3SecretValue.java | 8 +-
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 36 +-
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 128 ++--
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 3 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 4 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 8 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 25 -
.../om/protocolPB/grpc/GrpcClientConstants.java | 6 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 43 +-
.../request/validation/RegisterValidator.java | 39 ++
.../request/validation/RequestProcessingPhase.java | 2 +-
.../ozone/request/validation/package-info.java | 41 ++
.../hadoop/ozone/security/GDPRSymmetricKey.java | 10 +-
.../security/OzoneDelegationTokenSelector.java | 5 +-
.../ozone/security/OzoneTokenIdentifier.java | 2 +-
.../ozone/security/acl/IAccessAuthorizer.java | 26 +-
.../hadoop/ozone/security/acl/OzoneAclConfig.java | 76 --
.../ozone/snapshot/SnapshotDiffReportOzone.java | 13 +-
.../ozone/snapshot/SnapshotDiffResponse.java | 78 +--
.../org/apache/hadoop/ozone/util/RadixNode.java | 14 +-
.../org/apache/hadoop/ozone/util/RadixTree.java | 10 +-
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 8 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 2 -
.../ozone/client/io/TestSelectorOutputStream.java | 2 -
.../ozone/om/helpers/TestOmSnapshotInfo.java | 9 +-
.../hadoop/ozone/om/helpers/TestOzoneAclUtil.java | 11 +-
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 2 -
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 22 +-
.../hadoop/ozone/protocolPB/TestOMPBHelper.java | 95 +++
hadoop-ozone/datanode/pom.xml | 10 -
hadoop-ozone/dev-support/checks/acceptance.sh | 3 +
.../dist/dev-support/bin/dist-layout-stitching | 12 +-
hadoop-ozone/dist/pom.xml | 6 +-
.../dashboards/Ozone - DeleteKey Metrics.json | 2 +-
.../dist/src/main/compose/common/replicas-test.sh | 51 +-
.../compose/ozonesecure-ha/docker-compose.yaml | 90 +--
.../compose/ozonesecure-ha/s3g-virtual-host.yaml | 2 +-
.../compose/ozonesecure-ha/scm-decommission.yaml | 22 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 19 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 5 +-
.../dist/src/main/compose/xcompat/clients.yaml | 4 +
hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 2 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 1 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 8 +-
.../smoketest/debug/ozone-debug-tests-ec3-2.robot | 5 +
.../smoketest/debug/ozone-debug-tests-ec6-3.robot | 5 +
.../main/smoketest/debug/ozone-debug-tests.robot | 12 +-
.../src/main/smoketest/ozonefs/ozonefs-obs.robot | 7 +-
.../dist/src/main/smoketest/s3/bucketlist.robot | 10 +-
.../dist/src/main/smoketest/s3/objecthead.robot | 1 +
hadoop-ozone/dist/src/shell/conf/log4j.properties | 38 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 7 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 10 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 4 +-
.../hadoop/ozone/loadgenerators/LoadGenerator.java | 5 +-
.../freon/dev-support/findbugsExcludeFile.xml | 16 +
hadoop-ozone/{tools => freon}/pom.xml | 95 +--
.../ozone/freon/AbstractOmBucketReadWriteOps.java | 0
.../hadoop/ozone/freon/BaseAppendLogGenerator.java | 0
.../hadoop/ozone/freon/BaseFreonGenerator.java | 0
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 0
.../ozone/freon/ClosedContainerReplicator.java | 0
.../hadoop/ozone/freon/ContentGenerator.java | 0
.../hadoop/ozone/freon/DNRPCLoadGenerator.java | 0
.../hadoop/ozone/freon/DatanodeBlockPutter.java | 0
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 0
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 0
.../ozone/freon/DatanodeSimulationState.java | 0
.../hadoop/ozone/freon/DatanodeSimulator.java | 0
.../freon/FollowerAppendLogEntryGenerator.java | 0
.../java/org/apache/hadoop/ozone/freon/Freon.java | 0
.../apache/hadoop/ozone/freon/FreonHttpServer.java | 0
.../ozone/freon/FreonReplicationOptions.java | 0
.../apache/hadoop/ozone/freon/FreonSubcommand.java | 0
.../ozone/freon/HadoopBaseFreonGenerator.java | 0
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 0
.../hadoop/ozone/freon/HadoopFsGenerator.java | 0
.../hadoop/ozone/freon/HadoopFsValidator.java | 0
.../ozone/freon/HadoopNestedDirGenerator.java | 0
.../apache/hadoop/ozone/freon/HsyncGenerator.java | 0
.../hadoop/ozone/freon/KeyGeneratorUtil.java | 0
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 7 +-
.../hadoop/ozone/freon/OmBucketGenerator.java | 0
.../ozone/freon/OmBucketReadWriteFileOps.java | 0
.../ozone/freon/OmBucketReadWriteKeyOps.java | 0
.../apache/hadoop/ozone/freon/OmBucketRemover.java | 0
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 0
.../hadoop/ozone/freon/OmMetadataGenerator.java | 42 +-
.../hadoop/ozone/freon/OmRPCLoadGenerator.java | 0
.../hadoop/ozone/freon/OzoneClientCreator.java | 0
.../ozone/freon/OzoneClientKeyGenerator.java | 0
.../freon/OzoneClientKeyReadWriteListOps.java | 18 +-
.../hadoop/ozone/freon/OzoneClientKeyRemover.java | 0
.../ozone/freon/OzoneClientKeyValidator.java | 0
.../org/apache/hadoop/ozone/freon/PathSchema.java | 0
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 4 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 14 +-
.../hadoop/ozone/freon/RangeKeysGenerator.java | 0
.../hadoop/ozone/freon/S3BucketGenerator.java | 0
.../hadoop/ozone/freon/S3EntityGenerator.java | 0
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 0
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 18 +-
.../apache/hadoop/ozone/freon/SameKeyReader.java | 0
.../hadoop/ozone/freon/StorageSizeConverter.java | 0
.../hadoop/ozone/freon/StreamingGenerator.java | 0
.../freon/containergenerator/BaseGenerator.java | 0
.../containergenerator/GeneratorDatanode.java | 0
.../freon/containergenerator/GeneratorOm.java | 31 +-
.../freon/containergenerator/GeneratorScm.java | 0
.../freon/containergenerator/package-info.java | 0
.../apache/hadoop/ozone/freon/package-info.java | 0
.../hadoop/ozone/freon/TestContentGenerator.java | 0
.../apache/hadoop/ozone/freon/TestProgressBar.java | 0
.../containergenerator/TestGeneratorDatanode.java | 0
.../apache/hadoop/ozone/freon/package-info.java | 0
hadoop-ozone/httpfsgateway/pom.xml | 16 +-
.../apache/ozone/fs/http/server/HttpFSServer.java | 16 +-
.../org/apache/ozone/fs/http/server/JsonUtil.java | 7 +-
.../java/org/apache/ozone/lib/lang/XException.java | 32 +-
.../java/org/apache/ozone/lib/server/Server.java | 75 +-
.../service/hadoop/FileSystemAccessService.java | 32 +-
.../fs/http/server/metrics/TestHttpFSMetrics.java | 30 +-
hadoop-ozone/insight/pom.xml | 2 +-
hadoop-ozone/integration-test-s3/pom.xml | 25 +
.../apache/hadoop/ozone/s3/S3ClientFactory.java | 27 +-
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 163 +++--
.../hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java | 2 -
.../awssdk/v1/TestS3SDKV1WithRatisStreaming.java | 2 -
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 200 +++++-
.../hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java | 2 -
.../awssdk/v2/TestS3SDKV2WithRatisStreaming.java | 2 -
hadoop-ozone/integration-test/pom.xml | 6 +-
.../hadoop/conf/TestConfigurationFieldsBase.java | 2 +-
.../AbstractContractGetFileStatusTest.java | 9 +-
.../fs/contract/AbstractContractSeekTest.java | 2 +-
.../hadoop/fs/contract/ContractTestUtils.java | 4 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 10 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 77 +-
.../AbstractRootedOzoneFileSystemTestWithFSO.java | 1 -
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 2 -
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 2 -
.../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 6 +-
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 2 -
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 2 -
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 2 -
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 2 -
.../fs/ozone/TestOzoneFileSystemMetrics.java | 2 -
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 2 -
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 2 -
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 2 -
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 2 -
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 2 -
.../org/apache/hadoop/fs/ozone/TestSafeMode.java | 2 -
.../hadoop/hdds/scm/TestAllocateContainer.java | 2 -
.../apache/hadoop/hdds/scm/TestCommitInRatis.java | 2 -
.../hadoop/hdds/scm/TestContainerOperations.java | 4 +-
.../hdds/scm/TestContainerReportWithKeys.java | 2 -
.../hadoop/hdds/scm/TestContainerSmallFile.java | 2 -
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 2 -
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 9 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 2 -
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 4 +-
.../hdds/scm/TestSCMInstallSnapshotWithHA.java | 2 -
.../org/apache/hadoop/hdds/scm/TestSCMMXBean.java | 2 -
.../hadoop/hdds/scm/TestSCMNodeManagerMXBean.java | 2 -
.../hadoop/hdds/scm/TestSecretKeySnapshot.java | 2 -
.../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 3 -
.../hdds/scm/TestStorageContainerManager.java | 4 +-
.../hdds/scm/TestStorageContainerManagerHA.java | 2 -
...estStorageContainerManagerHAWithAllRunning.java | 2 -
.../apache/hadoop/hdds/scm/TestWatchForCommit.java | 2 +-
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 4 -
.../hadoop/hdds/scm/TestXceiverClientManager.java | 2 -
.../hadoop/hdds/scm/TestXceiverClientMetrics.java | 2 -
.../container/TestScmApplyTransactionFailure.java | 4 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 -
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 17 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 2 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 -
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 9 +
.../hdds/scm/pipeline/TestPipelineClose.java | 4 +-
.../scm/pipeline/TestPipelineManagerMXBean.java | 2 -
.../TestRatisPipelineCreateAndDestroy.java | 13 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 2 -
.../hdds/scm/pipeline/TestSCMPipelineMetrics.java | 2 -
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 2 -
.../safemode/TestSCMSafeModeWithPipelineRules.java | 20 +-
.../hadoop/hdds/scm/storage/TestCommitWatcher.java | 2 -
.../hdds/scm/storage/TestContainerCommandsEC.java | 4 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 10 +-
.../hadoop/hdds/upgrade/TestHddsUpgradeUtils.java | 7 +-
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 4 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +-
.../org/apache/hadoop/ozone/TestBlockTokens.java | 3 -
.../apache/hadoop/ozone/TestBlockTokensCLI.java | 3 -
.../ozone/TestContainerBalancerOperations.java | 2 -
.../apache/hadoop/ozone/TestDelegationToken.java | 2 -
.../ozone/TestGetClusterTreeInformation.java | 2 -
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +-
.../hadoop/ozone/TestMultipartObjectGet.java | 224 ------
.../apache/hadoop/ozone/TestOMSortDatanodes.java | 10 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 2 -
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 -
.../ozone/admin/om/lease/TestLeaseRecoverer.java | 3 -
.../ozone/client/rpc/OzoneRpcClientTests.java | 10 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 2 -
.../client/rpc/TestBlockDataStreamOutput.java | 2 -
.../ozone/client/rpc/TestBlockOutputStream.java | 2 -
.../rpc/TestBlockOutputStreamWithFailures.java | 2 -
.../rpc/TestCloseContainerHandlingByClient.java | 2 -
.../client/rpc/TestContainerStateMachine.java | 2 -
.../TestContainerStateMachineFailureOnRead.java | 3 +-
.../rpc/TestContainerStateMachineFailures.java | 7 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../rpc/TestContainerStateMachineStream.java | 2 -
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 2 +-
.../client/rpc/TestFailureHandlingByClient.java | 2 -
.../rpc/TestFailureHandlingByClientFlushDelay.java | 2 -
.../client/rpc/TestHybridPipelineOnDatanode.java | 2 -
.../rpc/TestMultiBlockWritesWithDnFailures.java | 2 -
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 4 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 2 -
.../rpc/TestOzoneClientRetriesOnExceptions.java | 2 -
.../ozone/client/rpc/TestOzoneRpcClient.java | 2 -
.../hadoop/ozone/client/rpc/TestReadRetries.java | 2 -
.../ozone/client/rpc/read/TestInputStreamBase.java | 4 +-
.../ozone/container/TestContainerReplication.java | 2 -
.../apache/hadoop/ozone/container/TestHelper.java | 2 +-
.../TestCloseContainerByPipeline.java | 4 +-
.../commandhandler/TestCloseContainerHandler.java | 2 -
.../commandhandler/TestDeleteContainerHandler.java | 2 -
.../commandhandler/TestFinalizeBlock.java | 2 -
.../TestRefreshVolumeUsageHandler.java | 2 -
.../container/metrics/TestContainerMetrics.java | 2 -
.../metrics/TestDatanodeQueueMetrics.java | 2 -
.../container/ozoneimpl/TestOzoneContainer.java | 2 -
.../ozoneimpl/TestOzoneContainerWithTLS.java | 2 -
.../ozoneimpl/TestSecureOzoneContainer.java | 2 -
.../replication/TestContainerReplication.java | 2 -
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 -
.../TestContainerScannerIntegrationAbstract.java | 2 -
.../TestDatanodeHddsVolumeFailureDetection.java | 2 -
.../TestDatanodeHddsVolumeFailureToleration.java | 2 -
.../freon/TestDataValidateWithDummyContainers.java | 3 -
.../TestDataValidateWithSafeByteOperations.java | 3 -
.../TestDataValidateWithUnsafeByteOperations.java | 3 -
.../freon/TestFreonWithDatanodeFastRestart.java | 2 -
.../ozone/freon/TestFreonWithPipelineDestroy.java | 2 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 2 -
.../hadoop/ozone/fsck/TestContainerMapper.java | 11 -
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 2 -
.../ozone/om/TestBucketLayoutWithOlderClient.java | 2 -
.../apache/hadoop/ozone/om/TestBucketOwner.java | 2 -
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 143 +++-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 2 -
.../org/apache/hadoop/ozone/om/TestListKeys.java | 2 -
.../hadoop/ozone/om/TestListKeysWithFSO.java | 25 +-
.../org/apache/hadoop/ozone/om/TestListStatus.java | 2 -
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 5 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 -
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 12 +-
.../ozone/om/TestOMStartupWithBucketLayout.java | 2 -
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 3 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 2 -
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 2 -
.../ozone/om/TestObjectStoreWithLegacyFS.java | 2 -
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 2 -
.../hadoop/ozone/om/TestOmBlockVersioning.java | 2 -
.../ozone/om/TestOmContainerLocationCache.java | 2 -
.../org/apache/hadoop/ozone/om/TestOmInit.java | 4 -
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 -
.../ozone/om/TestOzoneManagerConfiguration.java | 2 -
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 2 -
.../ozone/om/TestOzoneManagerListVolumes.java | 2 -
.../om/TestOzoneManagerListVolumesSecure.java | 2 -
.../ozone/om/TestOzoneManagerRestInterface.java | 2 -
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 2 -
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 -
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 22 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 2 -
.../hadoop/ozone/om/TestSecureOzoneManager.java | 2 -
.../om/multitenant/TestMultiTenantVolume.java | 4 +-
.../ozone/om/service/TestRangerBGSyncService.java | 2 -
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 65 +-
.../ozone/om/snapshot/TestOmSnapshotDisabled.java | 2 -
.../om/snapshot/TestOmSnapshotDisabledRestart.java | 2 -
.../om/snapshot/TestOmSnapshotFileSystem.java | 2 -
.../om/snapshot/TestOmSnapshotFileSystemFso.java | 3 -
...stOmSnapshotFileSystemFsoWithLinkedBuckets.java | 3 -
.../snapshot/TestOmSnapshotFileSystemLegacy.java | 3 -
...mSnapshotFileSystemLegacyWithLinkedBuckets.java | 3 -
.../snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 -
...mSnapshotFsoWithNativeLibWithLinkedBuckets.java | 2 -
.../TestOmSnapshotFsoWithoutNativeLib.java | 3 -
...apshotFsoWithoutNativeLibWithLinkedBuckets.java | 3 -
.../om/snapshot/TestOmSnapshotObjectStore.java | 3 -
...TestOmSnapshotObjectStoreWithLinkedBuckets.java | 3 -
.../TestOmSnapshotWithBucketLinkingLegacy.java | 3 -
.../TestOmSnapshotWithoutBucketLinkingLegacy.java | 3 -
.../om/snapshot/TestOzoneManagerHASnapshot.java | 25 +-
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 114 ++-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 2 -
.../om/snapshot/TestOzoneSnapshotRestore.java | 2 -
.../snapshot/TestSnapshotBackgroundServices.java | 2 -
...TestSnapshotDeletingServiceIntegrationTest.java | 2 -
.../TestSnapshotDirectoryCleaningService.java | 11 +-
.../ozone/parser/TestOzoneHARatisLogParser.java | 2 -
.../apache/hadoop/ozone/recon/ReconService.java | 109 +++
.../ozone/recon/TestReconAndAdminContainerCLI.java | 20 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 20 +-
.../ozone/recon/TestReconContainerEndpoint.java | 16 +-
.../TestReconInsightsForDeletedDirectories.java | 28 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 42 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 17 +-
.../ozone/recon/TestReconWithOzoneManager.java | 30 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 21 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 10 +-
.../ozone/reconfig/ReconfigurationTestBase.java | 2 -
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 7 +-
.../hadoop/ozone/shell/TestOzoneDatanodeShell.java | 2 -
.../hadoop/ozone/shell/TestOzoneDebugShell.java | 27 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 3 -
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 5 -
.../hadoop/ozone/shell/TestReconfigShell.java | 2 -
.../shell/TestReplicationConfigPreference.java | 2 -
.../tools/contract/AbstractContractDistCpTest.java | 20 +-
.../src/main/proto/OmClientProtocol.proto | 12 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 5 +
.../apache/hadoop/ozone/om/codec/package-info.java | 21 -
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 8 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 30 -
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 113 +--
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 23 +-
hadoop-ozone/ozone-manager/pom.xml | 19 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 6 +-
.../hadoop/ozone/om/DeletingServiceMetrics.java | 65 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 22 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 79 ++-
.../hadoop/ozone/om/OMMultiTenantManager.java | 19 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 24 +-
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 18 +-
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 6 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 471 ++++---------
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 52 ++
.../apache/hadoop/ozone/om/OmSnapshotMetrics.java | 68 +-
.../hadoop/ozone/om/OzoneListStatusHelper.java | 22 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 167 +++--
.../hadoop/ozone/om/SstFilteringService.java | 5 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 444 +++++++-----
.../ozone/om/codec/TokenIdentifierCodec.java | 28 +-
.../org/apache/hadoop/ozone/om/ha/OMHAMetrics.java | 15 +-
.../hadoop/ozone/om/helpers/OMAuditLogger.java | 8 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 91 ++-
.../om/ratis/OzoneManagerRatisServerConfig.java | 16 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 7 +-
.../om/request/bucket/OMBucketCreateRequest.java | 4 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 2 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 2 +-
.../om/request/file/OMDirectoryCreateRequest.java | 4 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 2 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 4 +-
.../request/file/OMFileCreateRequestWithFSO.java | 2 +-
.../om/request/key/OMAllocateBlockRequest.java | 2 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 2 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 4 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 2 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 2 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 22 +-
.../ozone/om/request/key/OMKeySetTimesRequest.java | 10 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 8 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 2 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 10 +-
.../request/key/acl/OMKeyAddAclRequestWithFSO.java | 8 +-
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 10 +-
.../key/acl/OMKeyRemoveAclRequestWithFSO.java | 8 +-
.../om/request/key/acl/OMKeySetAclRequest.java | 10 +-
.../request/key/acl/OMKeySetAclRequestWithFSO.java | 8 +-
.../S3InitiateMultipartUploadRequest.java | 4 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../multipart/S3MultipartUploadAbortRequest.java | 2 +-
.../S3MultipartUploadCommitPartRequest.java | 2 +-
.../S3MultipartUploadCompleteRequest.java | 12 +-
.../S3DeleteObjectTaggingRequestWithFSO.java | 4 +
.../tagging/S3PutObjectTaggingRequestWithFSO.java | 4 +
.../request/snapshot/OMSnapshotCreateRequest.java | 7 +-
.../snapshot/OMSnapshotSetPropertyRequest.java | 116 ++--
...alidator.java => OMClientVersionValidator.java} | 52 +-
...alidator.java => OMLayoutVersionValidator.java} | 49 +-
.../validation/RequestFeatureValidator.java | 1 +
.../om/request/validation/RequestValidations.java | 4 +-
.../om/request/validation/ValidatorRegistry.java | 5 +-
.../om/request/validation/VersionExtractor.java | 12 +-
.../om/request/volume/OMVolumeCreateRequest.java | 2 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 16 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 8 +-
.../volume/acl/OMVolumeRemoveAclRequest.java | 8 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 8 +-
.../om/response/bucket/OMBucketCreateResponse.java | 4 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 4 +-
.../response/bucket/OMBucketSetOwnerResponse.java | 2 +-
.../bucket/OMBucketSetPropertyResponse.java | 2 +-
.../response/bucket/acl/OMBucketAclResponse.java | 2 +-
.../response/file/OMDirectoryCreateResponse.java | 2 +-
.../file/OMDirectoryCreateResponseWithFSO.java | 2 +-
.../om/response/file/OMFileCreateResponse.java | 4 +-
.../response/file/OMFileCreateResponseWithFSO.java | 6 +-
.../om/response/file/OMRecoverLeaseResponse.java | 4 +-
.../response/key/AbstractOMKeyDeleteResponse.java | 2 +-
.../om/response/key/OMAllocateBlockResponse.java | 4 +-
.../key/OMAllocateBlockResponseWithFSO.java | 4 +-
.../key/OMDirectoriesPurgeResponseWithFSO.java | 10 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 8 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 8 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 6 +-
.../response/key/OMKeyCreateResponseWithFSO.java | 6 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 8 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 12 +-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 4 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 4 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 6 +-
.../om/response/key/OMKeySetTimesResponse.java | 2 +-
.../response/key/OMKeySetTimesResponseWithFSO.java | 4 +-
.../om/response/key/OMKeysDeleteResponse.java | 8 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 12 +-
.../om/response/key/OMKeysRenameResponse.java | 4 +-
.../om/response/key/OMOpenKeysDeleteResponse.java | 8 +-
.../om/response/key/acl/OMKeyAclResponse.java | 2 +-
.../response/key/acl/OMKeyAclResponseWithFSO.java | 4 +-
.../key/acl/prefix/OMPrefixAclResponse.java | 2 +-
.../AbstractS3MultipartAbortResponse.java | 12 +-
.../S3ExpiredMultipartUploadsAbortResponse.java | 12 +-
.../S3InitiateMultipartUploadResponse.java | 6 +-
.../S3InitiateMultipartUploadResponseWithFSO.java | 8 +-
.../multipart/S3MultipartUploadAbortResponse.java | 10 +-
.../S3MultipartUploadAbortResponseWithFSO.java | 8 +-
.../S3MultipartUploadCommitPartResponse.java | 10 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 8 +-
.../S3MultipartUploadCompleteResponse.java | 12 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 12 +-
.../response/s3/security/OMSetSecretResponse.java | 2 +-
.../response/s3/security/S3GetSecretResponse.java | 2 +-
.../s3/security/S3RevokeSecretResponse.java | 2 +-
.../s3/tagging/S3DeleteObjectTaggingResponse.java | 2 +-
.../S3DeleteObjectTaggingResponseWithFSO.java | 2 +-
.../s3/tagging/S3PutObjectTaggingResponse.java | 2 +-
.../tagging/S3PutObjectTaggingResponseWithFSO.java | 2 +-
.../tenant/OMSetRangerServiceVersionResponse.java | 2 +-
.../s3/tenant/OMTenantAssignAdminResponse.java | 2 +-
.../tenant/OMTenantAssignUserAccessIdResponse.java | 6 +-
.../response/s3/tenant/OMTenantCreateResponse.java | 4 +-
.../response/s3/tenant/OMTenantDeleteResponse.java | 4 +-
.../s3/tenant/OMTenantRevokeAdminResponse.java | 2 +-
.../tenant/OMTenantRevokeUserAccessIdResponse.java | 6 +-
.../security/OMCancelDelegationTokenResponse.java | 2 +-
.../security/OMGetDelegationTokenResponse.java | 2 +-
.../security/OMRenewDelegationTokenResponse.java | 2 +-
.../snapshot/OMSnapshotCreateResponse.java | 6 +-
.../snapshot/OMSnapshotDeleteResponse.java | 2 +-
.../OMSnapshotMoveDeletedKeysResponse.java | 2 +-
.../snapshot/OMSnapshotMoveTableKeysResponse.java | 2 +-
.../response/snapshot/OMSnapshotPurgeResponse.java | 2 +-
.../snapshot/OMSnapshotRenameResponse.java | 2 +-
.../snapshot/OMSnapshotSetPropertyResponse.java | 17 +-
.../response/upgrade/OMCancelPrepareResponse.java | 2 +-
.../upgrade/OMFinalizeUpgradeResponse.java | 2 +-
.../om/response/upgrade/OMPrepareResponse.java | 2 +-
.../om/response/volume/OMQuotaRepairResponse.java | 4 +-
.../om/response/volume/OMVolumeAclOpResponse.java | 2 +-
.../om/response/volume/OMVolumeCreateResponse.java | 2 +-
.../om/response/volume/OMVolumeDeleteResponse.java | 2 +-
.../response/volume/OMVolumeSetOwnerResponse.java | 2 +-
.../response/volume/OMVolumeSetQuotaResponse.java | 2 +-
.../hadoop/ozone/om/s3/S3SecretCacheProvider.java | 11 +-
.../om/service/AbstractKeyDeletingService.java | 20 +-
.../ozone/om/service/OMRangerBGSyncService.java | 157 +++--
.../ozone/om/service/OpenKeyCleanupService.java | 2 +-
.../ozone/om/service/SnapshotDeletingService.java | 66 +-
.../service/SnapshotDirectoryCleaningService.java | 4 +-
.../hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 4 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 19 +-
.../hadoop/ozone/om/snapshot/SnapshotUtils.java | 68 +-
.../om/snapshot/filter/ReclaimableDirFilter.java | 73 ++
.../om/snapshot/filter/ReclaimableFilter.java | 245 +++++++
.../om/snapshot/filter/ReclaimableKeyFilter.java | 171 +++++
.../filter/ReclaimableRenameEntryFilter.java | 93 +++
.../ozone/om/snapshot/filter}/package-info.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 2 +-
.../hadoop/ozone/security/AWSV4AuthValidator.java | 6 +-
.../ozone/om/TestGrpcOzoneManagerServer.java | 6 -
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 8 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 55 +-
.../hadoop/ozone/om/TestOmSnapshotManager.java | 105 ++-
.../ozone/om/lock/TestOzoneLockProvider.java | 6 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 4 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +
.../om/ratis/TestOzoneManagerRatisRequest.java | 2 -
.../ozone/om/request/bucket/TestBucketRequest.java | 2 +
.../request/file/TestOMDirectoryCreateRequest.java | 4 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 2 +
.../om/request/key/TestOMKeyCreateRequest.java | 2 +
.../ozone/om/request/key/TestOMKeyRequest.java | 2 +
.../s3/multipart/TestS3MultipartRequest.java | 2 +
.../TestS3DeleteObjectTaggingRequestWithFSO.java | 2 +-
.../TestS3PutObjectTaggingRequestWithFSO.java | 2 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 54 ++
.../validation/TestOMValidatorProcessor.java | 583 ++++++++++++++++
.../TestRequestFeatureValidatorProcessor.java | 524 --------------
.../request/validation/TestValidatorRegistry.java | 4 +-
.../request/validation/TestVersionExtractor.java | 17 +-
.../GeneralValidatorsForTesting.java | 8 +-
.../ValidatorsForOnlyOldClientValidations.java | 2 +-
.../om/request/volume/TestOMVolumeRequest.java | 2 +
.../ozone/om/service/TestCompactionService.java | 4 -
.../ozone/om/service/TestKeyDeletingService.java | 2 -
.../service/TestMultipartUploadCleanupService.java | 2 -
.../om/service/TestOpenKeyCleanupService.java | 3 -
.../ozone/om/snapshot/TestMultiSnapshotLocks.java | 2 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 44 +-
.../ozone/om/snapshot/TestSstFilteringService.java | 18 +-
.../filter/AbstractReclaimableFilterTest.java | 324 +++++++++
.../snapshot/filter/TestReclaimableDirFilter.java | 143 ++++
.../om/snapshot/filter/TestReclaimableFilter.java | 291 ++++++++
.../snapshot/filter/TestReclaimableKeyFilter.java | 318 +++++++++
.../filter/TestReclaimableRenameEntryFilter.java | 204 ++++++
.../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 4 +-
hadoop-ozone/ozonefs-common/pom.xml | 15 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 17 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 4 +-
.../java/org/apache/hadoop/fs/ozone/Statistic.java | 6 +-
hadoop-ozone/ozonefs/pom.xml | 7 +-
hadoop-ozone/pom.xml | 263 +------
.../recon/schema/ContainerSchemaDefinition.java | 30 +-
.../ozone/recon/schema/ReconSqlDbConfig.java | 176 ++---
.../org/apache/ozone/recon/schema/SqlDbUtils.java | 30 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../apache/hadoop/ozone/recon/ReconConstants.java | 8 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 25 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 6 +-
.../ozone/recon/api/ClusterStateEndpoint.java | 17 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 84 +--
.../ozone/recon/api/OMDBInsightEndpoint.java | 8 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 14 +-
.../recon/api/types/ContainerKeyPrefixImpl.java | 9 +-
.../ozone/recon/api/types/DatanodeMetadata.java | 34 +-
.../ozone/recon/api/types/DatanodesResponse.java | 8 +-
.../recon/api/types/KeyEntityInfoProtoWrapper.java | 16 +-
.../hadoop/ozone/recon/api/types/NSSummary.java | 11 +
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 30 +-
.../ozone/recon/fsck/ReconSafeModeMgrTask.java | 9 +-
.../recon/metrics/ContainerHealthMetrics.java | 28 +-
.../recon/metrics/OzoneManagerSyncMetrics.java | 30 +-
.../recon/recovery/ReconOmMetadataManagerImpl.java | 14 +-
.../recon/scm/ContainerReplicaHistoryList.java | 8 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 127 +---
.../ozone/recon/scm/ReconPolicyProvider.java | 14 +-
.../ozone/recon/scm/ReconSCMDBDefinition.java | 12 +-
.../scm/ReconStorageContainerManagerFacade.java | 34 +-
.../recon/spi/impl/KeyPrefixContainerCodec.java | 4 +-
.../ozone/recon/spi/impl/ReconDBDefinition.java | 10 +-
.../ozone/recon/tasks/FileSizeCountTaskFSO.java | 5 +-
.../ozone/recon/tasks/FileSizeCountTaskOBS.java | 5 +-
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 4 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 2 +-
.../ozone/recon/tasks/NSSummaryTaskWithOBS.java | 2 +-
.../ozone/recon/tasks/OmTableInsightTask.java | 6 +-
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 48 +-
.../upgrade/InitialConstraintUpgradeAction.java | 5 +-
.../recon/upgrade/ReconLayoutVersionManager.java | 17 +-
.../upgrade/ReconTaskStatusTableUpgradeAction.java | 11 +-
.../ozone/recon/upgrade/ReconUpgradeAction.java | 4 +-
.../UnhealthyContainerReplicaMismatchAction.java | 7 +-
.../webapps/recon/ozone-recon-web/package.json | 2 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 174 +++--
.../ozone/recon/OMMetadataManagerTestUtils.java | 8 +-
.../ozone/recon/api/TestContainerEndpoint.java | 3 +-
.../TestSchemaVersionTableDefinition.java | 10 +-
.../ozone/recon/scm/TestReconNodeManager.java | 13 +-
.../recon/tasks/AbstractNSSummaryTaskTest.java | 767 ++++++++++++++++++++
.../ozone/recon/tasks/TestFileSizeCountTask.java | 24 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 325 +--------
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 328 ++-------
.../recon/tasks/TestNSSummaryTaskWithLegacy.java | 408 +----------
.../TestNSSummaryTaskWithLegacyOBSLayout.java | 316 +--------
.../recon/tasks/TestNSSummaryTaskWithOBS.java | 305 +-------
.../ozone/recon/tasks/TestOmTableInsightTask.java | 14 +-
.../TestInitialConstraintUpgradeAction.java | 2 +-
.../upgrade/TestReconLayoutVersionManager.java | 41 +-
hadoop-ozone/s3gateway/pom.xml | 32 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 17 +-
.../ozone/s3/endpoint/ListBucketResponse.java | 11 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 13 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 4 +-
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 24 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 14 +-
.../ozone/s3/signature/StringToSignProducer.java | 4 +-
.../apache/hadoop/ozone/s3/util/RFC1123Util.java | 7 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 10 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 9 +
.../hadoop/ozone/client/ObjectStoreStub.java | 8 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 6 -
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 6 +
hadoop-ozone/tools/pom.xml | 34 +-
.../parser/ContainerDatanodeDatabase.java | 235 -------
.../hadoop/ozone/containerlog/parser/DBConsts.java | 38 -
.../ozone/debug/CompactionLogDagPrinter.java | 63 --
.../hadoop/ozone/debug/DBDefinitionFactory.java | 19 +-
.../debug/audit/parser/common/DatabaseHelper.java | 11 +-
.../debug/audit/parser/common/ParserConsts.java | 8 +-
.../ozone/debug/container/ContainerLogParser.java | 93 ---
.../LogParser.java} | 15 +-
.../logs/container/ContainerInfoCommand.java} | 50 +-
.../logs/container/ContainerLogController.java | 82 +++
.../debug/logs/container/ContainerLogParser.java | 98 +++
.../container/DuplicateOpenContainersCommand.java} | 38 +-
.../ozone/debug/logs/container/ListContainers.java | 87 +++
.../debug/{ => logs}/container/package-info.java | 5 +-
.../container/utils/ContainerDatanodeDatabase.java | 773 +++++++++++++++++++++
.../container/utils}/ContainerLogFileParser.java | 18 +-
.../container/utils}/DatanodeContainerInfo.java | 2 +-
.../debug/logs/container/utils/SQLDBConstants.java | 146 ++++
.../logs/container/utils}/package-info.java | 2 +-
.../parser => debug/logs}/package-info.java | 4 +-
.../ozone/debug/om/CompactionLogDagPrinter.java | 111 +++
.../org/apache/hadoop/ozone/debug/om/OMDebug.java | 11 +
.../apache/hadoop/ozone/debug/om/PrefixParser.java | 38 +-
.../debug/replicas/BlockExistenceVerifier.java | 85 +++
.../debug/replicas/BlockVerificationResult.java | 62 ++
.../ozone/debug/replicas/ChecksumVerifier.java | 87 +++
.../hadoop/ozone/debug/replicas/Checksums.java | 163 -----
.../ozone/debug/replicas/ReplicaVerifier.java | 10 +-
.../ozone/debug/replicas/ReplicasVerify.java | 143 +++-
.../debug/replicas/chunk/ChunkDataNodeDetails.java | 43 --
.../ozone/debug/replicas/chunk/ChunkDetails.java | 53 --
.../debug/replicas/chunk/ChunkKeyHandler.java | 139 ++--
.../debug/replicas/chunk/ContainerChunkInfo.java | 105 ---
.../hadoop/ozone/repair/TransactionInfoRepair.java | 2 +-
.../schemaupgrade/ContainerUpgradeResult.java | 2 +-
.../schemaupgrade/VolumeUpgradeResult.java | 2 +-
.../hadoop/ozone/repair/om/FSORepairTool.java | 40 +-
.../hadoop/ozone/repair/om/OMRatisLogRepair.java | 217 ++++++
.../apache/hadoop/ozone/repair/om/OMRepair.java | 3 +-
.../src/main/java/org/apache/ozone/graph/Edge.java | 0
.../org/apache/ozone/graph/PrintableGraph.java | 40 +-
.../java/org/apache/ozone/graph/package-info.java | 0
.../resources/container-log-db-queries.properties | 24 -
.../ozone/repair/TestTransactionInfoRepair.java | 2 +-
.../org/apache/ozone/graph/TestPrintableGraph.java | 12 +-
pom.xml | 418 ++++++++++-
1102 files changed, 15117 insertions(+), 11368 deletions(-)
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
index 6f20f22a8b,1749d94ae9..682348ea20
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
@@@ -176,6 -160,6 +176,7 @@@ public final class ContainerLogger
"Index=" + containerData.getReplicaIndex(),
"BCSID=" + containerData.getBlockCommitSequenceId(),
"State=" + containerData.getState(),
- "Volume=" + containerData.getVolume());
++ "Volume=" + containerData.getVolume(),
+ "DataChecksum=" + checksumToString(containerData.getDataChecksum()));
}
}
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 86817ef258,7927864861..da1bb06471
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@@ -17,91 -17,50 +17,97 @@@
package org.apache.hadoop.ozone.container.keyvalue;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.QUASI_CLOSED;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.GB;
+import static org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager.getContainerChecksumFile;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.writeContainerDataTreeProto;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.WRITE_STAGE;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createBlockMetaData;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createDbInstancesForTestIfNeeded;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
+import static org.apache.hadoop.ozone.container.keyvalue.TestContainerCorruptions.getBlock;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+ import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.eq;
- import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.atMostOnce;
+ import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
+ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.common.collect.ImmutableList;
import java.io.File;
import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Random;
+import java.util.Set;
import java.util.UUID;
+ import java.util.concurrent.CompletableFuture;
+ import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.client.BlockID;
+ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.common.Checksum;
+import org.apache.hadoop.ozone.common.ChecksumData;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
+import org.apache.hadoop.ozone.container.checksum.DNContainerOperationClient;
import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@@@ -115,36 -72,29 +121,39 @@@ import org.apache.hadoop.ozone.containe
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.util.Sets;
import org.apache.hadoop.util.Time;
+ import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
- import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.MockedStatic;
import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
/**
* Unit tests for {@link KeyValueHandler}.
*/
- @Timeout(300)
public class TestKeyValueHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(TestKeyValueHandler.class);
+
@TempDir
private Path tempDir;
-
- private static final String DATANODE_UUID = UUID.randomUUID().toString();
+ @TempDir
+ private Path dbFile;
private static final long DUMMY_CONTAINER_ID = 9999;
private static final String DUMMY_PATH = "dummy/dir/doesnt/exist";
@@@ -155,34 -101,12 +164,35 @@@
private HddsDispatcher dispatcher;
private KeyValueHandler handler;
+ private OzoneConfiguration conf;
+ private long maxContainerSize;
+ /**
+ * Number of corrupt blocks and chunks.
+ */
+ public static Stream<Arguments> corruptionValues() {
+ return Stream.of(
+ Arguments.of(5, 0),
+ Arguments.of(0, 5),
+ Arguments.of(0, 10),
+ Arguments.of(10, 0),
+ Arguments.of(5, 10),
+ Arguments.of(10, 5),
+ Arguments.of(2, 3),
+ Arguments.of(3, 2),
+ Arguments.of(4, 6),
+ Arguments.of(6, 4),
+ Arguments.of(6, 9),
+ Arguments.of(9, 6)
+ );
+ }
+
@BeforeEach
- public void setup() throws StorageContainerException {
- OzoneConfiguration conf = new OzoneConfiguration();
+ public void setup() throws IOException {
// Create mock HddsDispatcher and KeyValueHandler.
+ conf = new OzoneConfiguration();
+ conf.set(HDDS_DATANODE_DIR_KEY, tempDir.toString());
+ conf.set(OZONE_METADATA_DIRS, tempDir.toString());
handler = mock(KeyValueHandler.class);
HashMap<ContainerType, Handler> handlers = new HashMap<>();
@@@ -427,6 -353,68 +441,68 @@@
"Close container should return Invalid container error");
}
+ @Test
+ public void testCreateContainerWithFailure() throws Exception {
+ final String testDir = tempDir.toString();
+ final long containerID = 1L;
+ final String clusterId = UUID.randomUUID().toString();
+ final String datanodeId = UUID.randomUUID().toString();
+ final ConfigurationSource conf = new OzoneConfiguration();
+ final ContainerSet containerSet = spy(newContainerSet());
+ final MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
+
+ HddsVolume hddsVolume = new HddsVolume.Builder(testDir).conf(conf)
+ .clusterID(clusterId).datanodeUuid(datanodeId)
+ .volumeSet(volumeSet)
+ .build();
+
+ hddsVolume.format(clusterId);
+ hddsVolume.createWorkingDir(clusterId, null);
+ hddsVolume.createTmpDirs(clusterId);
+
+ when(volumeSet.getVolumesList())
+ .thenReturn(Collections.singletonList(hddsVolume));
+
+ List<HddsVolume> hddsVolumeList = StorageVolumeUtil
+ .getHddsVolumesList(volumeSet.getVolumesList());
+
+ assertEquals(1, hddsVolumeList.size());
+
+ final ContainerMetrics metrics = ContainerMetrics.create(conf);
-
++
+ final AtomicInteger icrReceived = new AtomicInteger(0);
+
+ final KeyValueHandler kvHandler = new KeyValueHandler(conf,
+ datanodeId, containerSet, volumeSet, metrics,
- c -> icrReceived.incrementAndGet());
++ c -> icrReceived.incrementAndGet(), new ContainerChecksumTreeManager(conf));
+ kvHandler.setClusterID(clusterId);
-
++
+ final ContainerCommandRequestProto createContainer =
+ createContainerRequest(datanodeId, containerID);
+
+ Semaphore semaphore = new Semaphore(1);
+ doAnswer(invocation -> {
+ semaphore.acquire();
+ throw new StorageContainerException(ContainerProtos.Result.IO_EXCEPTION);
+ }).when(containerSet).addContainer(any());
+
+ semaphore.acquire();
- CompletableFuture.runAsync(() ->
++ CompletableFuture.runAsync(() ->
+ kvHandler.handleCreateContainer(createContainer, null)
+ );
+
+ // commit bytes has been allocated by volumeChoosingPolicy which is called in KeyValueContainer#create
+ GenericTestUtils.waitFor(() -> hddsVolume.getCommittedBytes() == maxContainerSize,
+ 1000, 50000);
+ semaphore.release();
+
+ LOG.info("Committed bytes: {}", hddsVolume.getCommittedBytes());
-
++
+ // release committed bytes as exception is thrown
+ GenericTestUtils.waitFor(() -> hddsVolume.getCommittedBytes() == 0,
+ 1000, 50000);
+ }
+
@Test
public void testDeleteContainer() throws IOException {
final String testDir = tempDir.toString();
@@@ -521,202 -509,60 +597,256 @@@
}
}
+ /**
+ * Tests that deleting a container decrements the cached used space of its volume.
+ */
+ @Test
+ public void testDeleteDecrementsVolumeUsedSpace() throws IOException {
+ final long containerID = 1;
+ final String clusterId = UUID.randomUUID().toString();
+ final String datanodeId = UUID.randomUUID().toString();
+ final ContainerSet containerSet = newContainerSet();
+ final MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
+ final HddsVolume hddsVolume = mock(HddsVolume.class);
+ when(hddsVolume.getDeletedContainerDir()).thenReturn(new File(""));
+
+ final ConfigurationSource conf = new OzoneConfiguration();
+ final ContainerMetrics metrics = ContainerMetrics.create(conf);
+ final AtomicInteger icrReceived = new AtomicInteger(0);
+ final long containerBytesUsed = 1024 * 1024;
+
+ // We're testing KeyValueHandler in this test, all the other objects are mocked
+ final KeyValueHandler kvHandler = new KeyValueHandler(conf,
+ datanodeId, containerSet, volumeSet, metrics,
- c -> icrReceived.incrementAndGet());
++ c -> icrReceived.incrementAndGet(), new ContainerChecksumTreeManager(conf));
+ kvHandler.setClusterID(clusterId);
+
+ // Setup ContainerData and Container mocks
+ KeyValueContainerData containerData = mock(KeyValueContainerData.class);
+ when(containerData.getContainerID()).thenReturn(containerID);
+ when(containerData.getVolume()).thenReturn(hddsVolume);
+ when(containerData.getBytesUsed()).thenReturn(containerBytesUsed);
+ when(containerData.getState()).thenReturn(ContainerProtos.ContainerDataProto.State.CLOSED);
+ when(containerData.isOpen()).thenReturn(false);
+ when(containerData.getLayoutVersion()).thenReturn(ContainerLayoutVersion.FILE_PER_BLOCK);
+ when(containerData.getDbFile()).thenReturn(new File(tempDir.toFile(), "dummy.db"));
+ when(containerData.getContainerPath()).thenReturn(tempDir.toString());
+ when(containerData.getMetadataPath()).thenReturn(tempDir.toString());
+
+ KeyValueContainer container = mock(KeyValueContainer.class);
+ when(container.getContainerData()).thenReturn(containerData);
+ when(container.hasBlocks()).thenReturn(true);
+
+ containerSet.addContainer(container);
+ assertNotNull(containerSet.getContainer(containerID));
+
+ // This is the method we're testing. It should decrement used space in the volume when deleting this container
+ kvHandler.deleteContainer(container, true);
+ assertNull(containerSet.getContainer(containerID));
+
+ // Verify ICR was sent (once for delete)
+ assertEquals(1, icrReceived.get(), "ICR should be sent for delete");
+ verify(container, times(1)).delete();
+ // Verify decrementUsedSpace was called with the correct amount
+ verify(hddsVolume, times(1)).decrementUsedSpace(eq(containerBytesUsed));
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void testContainerChecksumInvocation(ContainerLayoutVersion layoutVersion) throws Exception {
+ conf = new OzoneConfiguration();
+
+ KeyValueContainerData data = new KeyValueContainerData(123L, layoutVersion, GB,
+ PipelineID.randomId().toString(), randomDatanodeDetails().getUuidString());
+ data.setMetadataPath(tempDir.toString());
+ data.setDbFile(dbFile.toFile());
+
+ Container container = new KeyValueContainer(data, conf);
+ createBlockMetaData(data, 5, 3);
+ ContainerSet containerSet = newContainerSet();
+ containerSet.addContainer(container);
+
+ // Allows checking the invocation count of the lambda.
+ AtomicInteger icrCount = new AtomicInteger(0);
+ IncrementalReportSender<Container> icrSender = c -> {
+ // Check that the ICR contains expected info about the container.
+ ContainerReplicaProto report = c.getContainerReport();
+ long reportedID = report.getContainerID();
+ Assertions.assertEquals(container.getContainerData().getContainerID(), reportedID);
+
+ long reportDataChecksum = report.getDataChecksum();
+ assertNotEquals(0, reportDataChecksum,
+ "Container report should have populated the checksum field with a non-zero value.");
+ icrCount.incrementAndGet();
+ };
+
+ KeyValueHandler keyValueHandler = new KeyValueHandler(conf, randomDatanodeDetails().getUuidString(), containerSet,
+ mock(MutableVolumeSet.class), mock(ContainerMetrics.class), icrSender, new ContainerChecksumTreeManager(conf));
+
+ Assertions.assertEquals(0, icrCount.get());
+ // This should trigger container report validation in the ICR handler above.
+ DNContainerOperationClient mockDnClient = mock(DNContainerOperationClient.class);
+ DatanodeDetails peer1 = MockDatanodeDetails.randomDatanodeDetails();
+ DatanodeDetails peer2 = MockDatanodeDetails.randomDatanodeDetails();
+ DatanodeDetails peer3 = MockDatanodeDetails.randomDatanodeDetails();
+ when(mockDnClient.getContainerChecksumInfo(anyLong(), any())).thenReturn(null);
+ keyValueHandler.reconcileContainer(mockDnClient, container, Sets.newHashSet(peer1, peer2, peer3));
+ // Make sure all the replicas are used for reconciliation.
+ Mockito.verify(mockDnClient, atMostOnce()).getContainerChecksumInfo(anyLong(), eq(peer1));
+ Mockito.verify(mockDnClient, atMostOnce()).getContainerChecksumInfo(anyLong(), eq(peer2));
+ Mockito.verify(mockDnClient, atMostOnce()).getContainerChecksumInfo(anyLong(), eq(peer3));
+ Assertions.assertEquals(1, icrCount.get());
+ }
+
+ @ParameterizedTest
+ @MethodSource("corruptionValues")
+ public void testFullContainerReconciliation(int numBlocks, int numChunks) throws Exception {
+ KeyValueHandler kvHandler = createKeyValueHandler(tempDir);
+ ContainerChecksumTreeManager checksumManager = kvHandler.getChecksumManager();
+ DNContainerOperationClient dnClient = new DNContainerOperationClient(conf, null, null);
+ final long containerID = 100L;
+ // Create 3 containers with 15 blocks each and 3 replicas.
+ List<KeyValueContainer> containers = createContainerWithBlocks(kvHandler, containerID, 15, 3);
+ assertEquals(3, containers.size());
+
+ // Introduce corruption in each container on different replicas.
+ introduceCorruption(kvHandler, containers.get(1), numBlocks, numChunks, false);
+ introduceCorruption(kvHandler, containers.get(2), numBlocks, numChunks, true);
+
+ // Without reconciliation, checksums should be different because of the corruption.
+ Set<Long> checksumsBeforeReconciliation = new HashSet<>();
+ for (KeyValueContainer kvContainer : containers) {
+ Optional<ContainerProtos.ContainerChecksumInfo> containerChecksumInfo =
+ checksumManager.read(kvContainer.getContainerData());
+ assertTrue(containerChecksumInfo.isPresent());
+ long dataChecksum = containerChecksumInfo.get().getContainerMerkleTree().getDataChecksum();
+ assertEquals(kvContainer.getContainerData().getDataChecksum(), dataChecksum);
+ checksumsBeforeReconciliation.add(dataChecksum);
+ }
+ // There should be more than 1 checksum because of the corruption.
+ assertTrue(checksumsBeforeReconciliation.size() > 1);
+
+ List<DatanodeDetails> datanodes = ImmutableList.of(randomDatanodeDetails(), randomDatanodeDetails(),
+ randomDatanodeDetails());
+ Map<String, KeyValueContainer> dnToContainerMap = new HashMap<>();
+ dnToContainerMap.put(datanodes.get(0).getUuidString(), containers.get(0));
+ dnToContainerMap.put(datanodes.get(1).getUuidString(), containers.get(1));
+ dnToContainerMap.put(datanodes.get(2).getUuidString(), containers.get(2));
+
+ // Setup mock for each datanode network calls needed for reconciliation.
+ try (MockedStatic<ContainerProtocolCalls> containerProtocolMock =
+ Mockito.mockStatic(ContainerProtocolCalls.class)) {
+ mockContainerProtocolCalls(containerProtocolMock, dnToContainerMap, checksumManager, kvHandler, containerID);
+
+ kvHandler.reconcileContainer(dnClient, containers.get(0), Sets.newHashSet(datanodes));
+ kvHandler.reconcileContainer(dnClient, containers.get(1), Sets.newHashSet(datanodes));
+ kvHandler.reconcileContainer(dnClient, containers.get(2), Sets.newHashSet(datanodes));
+
+ // After reconciliation, checksums should be the same for all containers.
+ ContainerProtos.ContainerChecksumInfo prevContainerChecksumInfo = null;
+ for (KeyValueContainer kvContainer : containers) {
+ kvHandler.createContainerMerkleTree(kvContainer);
+ Optional<ContainerProtos.ContainerChecksumInfo> containerChecksumInfo =
+ checksumManager.read(kvContainer.getContainerData());
+ assertTrue(containerChecksumInfo.isPresent());
+ long dataChecksum = containerChecksumInfo.get().getContainerMerkleTree().getDataChecksum();
+ assertEquals(kvContainer.getContainerData().getDataChecksum(), dataChecksum);
+ if (prevContainerChecksumInfo != null) {
+ assertEquals(prevContainerChecksumInfo.getContainerMerkleTree().getDataChecksum(), dataChecksum);
+ }
+ prevContainerChecksumInfo = containerChecksumInfo.get();
+ }
+ }
+ }
+
+ private void mockContainerProtocolCalls(MockedStatic<ContainerProtocolCalls> containerProtocolMock,
+ Map<String, KeyValueContainer> dnToContainerMap,
+ ContainerChecksumTreeManager checksumManager,
+ KeyValueHandler kvHandler,
+ long containerID) {
+ // Mock getContainerChecksumInfo
+ containerProtocolMock.when(() -> ContainerProtocolCalls.getContainerChecksumInfo(any(), anyLong(), any()))
+ .thenAnswer(inv -> {
+ XceiverClientSpi xceiverClientSpi = inv.getArgument(0);
+ Pipeline pipeline = xceiverClientSpi.getPipeline();
+ assertEquals(1, pipeline.size());
+ DatanodeDetails dn = pipeline.getFirstNode();
+ KeyValueContainer container = dnToContainerMap.get(dn.getUuidString());
+ ByteString checksumInfo = checksumManager.getContainerChecksumInfo(container.getContainerData());
+ return ContainerProtos.GetContainerChecksumInfoResponseProto.newBuilder()
+ .setContainerID(containerID)
+ .setContainerChecksumInfo(checksumInfo)
+ .build();
+ });
+
+ // Mock getBlock
+ containerProtocolMock.when(() -> ContainerProtocolCalls.getBlock(any(), any(), any(), any(), anyMap()))
+ .thenAnswer(inv -> {
+ XceiverClientSpi xceiverClientSpi = inv.getArgument(0);
+ Pipeline pipeline = xceiverClientSpi.getPipeline();
+ assertEquals(1, pipeline.size());
+ DatanodeDetails dn = pipeline.getFirstNode();
+ KeyValueContainer container = dnToContainerMap.get(dn.getUuidString());
+ ContainerProtos.BlockData blockData = kvHandler.getBlockManager().getBlock(container, inv.getArgument(2))
+ .getProtoBufMessage();
+ return ContainerProtos.GetBlockResponseProto.newBuilder()
+ .setBlockData(blockData)
+ .build();
+ });
+
+ // Mock readChunk
+ containerProtocolMock.when(() -> ContainerProtocolCalls.readChunk(any(), any(), any(), any(), any()))
+ .thenAnswer(inv -> {
+ XceiverClientSpi xceiverClientSpi = inv.getArgument(0);
+ Pipeline pipeline = xceiverClientSpi.getPipeline();
+ assertEquals(1, pipeline.size());
+ DatanodeDetails dn = pipeline.getFirstNode();
+ KeyValueContainer container = dnToContainerMap.get(dn.getUuidString());
+ return createReadChunkResponse(inv, container, kvHandler);
+ });
+ }
+
+ // Helper method to create readChunk responses
+ private ContainerProtos.ReadChunkResponseProto createReadChunkResponse(InvocationOnMock inv,
+ KeyValueContainer container,
+ KeyValueHandler kvHandler) throws IOException {
+ ContainerProtos.DatanodeBlockID blockId = inv.getArgument(2);
+ ContainerProtos.ChunkInfo chunkInfo = inv.getArgument(1);
+ return ContainerProtos.ReadChunkResponseProto.newBuilder()
+ .setBlockID(blockId)
+ .setChunkData(chunkInfo)
+ .setData(kvHandler.getChunkManager().readChunk(container, BlockID.getFromProtobuf(blockId),
+ ChunkInfo.getFromProtoBuf(chunkInfo), null).toByteString())
+ .build();
+ }
+
+ @Test
+ public void testGetContainerChecksumInfoOnInvalidContainerStates() {
+ when(handler.handleGetContainerChecksumInfo(any(), any())).thenCallRealMethod();
+
+ // Only mock what is necessary for the request to fail. This test does not cover allowed states.
+ KeyValueContainer container = mock(KeyValueContainer.class);
+ KeyValueContainerData containerData = mock(KeyValueContainerData.class);
+ when(container.getContainerData()).thenReturn(containerData);
+
+ ContainerCommandRequestProto request = mock(ContainerCommandRequestProto.class);
+ when(request.hasGetContainerChecksumInfo()).thenReturn(true);
+ when(request.getCmdType()).thenReturn(ContainerProtos.Type.GetContainerChecksumInfo);
+ when(request.getTraceID()).thenReturn("123");
+
+ Set<State> disallowedStates = EnumSet.allOf(State.class);
+ disallowedStates.removeAll(EnumSet.of(CLOSED, QUASI_CLOSED, UNHEALTHY));
+
+ for (State state : disallowedStates) {
+ when(containerData.getState()).thenReturn(state);
+ ContainerProtos.ContainerCommandResponseProto response = handler.handleGetContainerChecksumInfo(request,
+ container);
+ assertNotNull(response);
+ assertEquals(ContainerProtos.Result.UNCLOSED_CONTAINER_IO, response.getResult());
+ assertTrue(response.getMessage().contains(state.toString()), "Response message did not contain the container " +
+ "state " + state);
+ }
+ }
+
@Test
public void testDeleteContainerTimeout() throws IOException {
final String testDir = tempDir.toString();
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
index aee25dbbb4,ce08cbb3f2..67bf1ad194
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
@@@ -41,19 -45,17 +45,19 @@@ public final class ContainerReplica imp
private final long keyCount;
private final long bytesUsed;
private final boolean isEmpty;
+ private final long dataChecksum;
private ContainerReplica(ContainerReplicaBuilder b) {
- containerID = b.containerID;
- state = b.state;
- datanodeDetails = b.datanode;
- placeOfBirth = Optional.ofNullable(b.placeOfBirth).orElse(datanodeDetails.getUuid());
- keyCount = b.keyCount;
- bytesUsed = b.bytesUsed;
- replicaIndex = b.replicaIndex;
- isEmpty = b.isEmpty;
- sequenceId = b.sequenceId;
- dataChecksum = b.dataChecksum;
+ this.containerID = Objects.requireNonNull(b.containerID, "containerID == null");
+ this.state = Objects.requireNonNull(b.state, "state == null");
+ this.datanodeDetails = Objects.requireNonNull(b.datanode, "datanode == null");
+ this.originDatanodeId = b.placeOfBirth;
+ this.keyCount = b.keyCount;
+ this.bytesUsed = b.bytesUsed;
+ this.replicaIndex = b.replicaIndex;
+ this.isEmpty = b.isEmpty;
+ this.sequenceId = b.sequenceId;
++ this.dataChecksum = b.dataChecksum;
}
public ContainerID getContainerID() {
@@@ -181,19 -179,16 +185,17 @@@
@Override
public String toString() {
- return "ContainerReplica{" +
- "containerID=" + containerID +
- ", state=" + state +
- ", datanodeDetails=" + datanodeDetails +
- ", placeOfBirth=" + placeOfBirth +
- ", sequenceId=" + sequenceId +
- ", keyCount=" + keyCount +
- ", bytesUsed=" + bytesUsed + ((replicaIndex > 0) ?
- ",replicaIndex=" + replicaIndex :
- "") +
- ", isEmpty=" + isEmpty +
- ", dataChecksum=" + dataChecksum +
- '}';
+ return "ContainerReplica{" + containerID
+ + " (" + state
+ + ") currentDN=" + datanodeDetails
+ + (originDatanodeId != null ? ", originDN=" + originDatanodeId : " (origin)")
+ + ", bcsid=" + sequenceId
+ + (replicaIndex > 0 ? ", replicaIndex=" + replicaIndex : "")
+ + ", keyCount=" + keyCount
+ + ", bytesUsed=" + bytesUsed
+ + ", " + (isEmpty ? "empty" : "non-empty")
++ + ", dataChecksum=" + dataChecksum +
+ + '}';
}
/**
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/reconciliation/ReconcileContainerEventHandler.java
index 468ba18b62,0000000000..9558e1d51d
mode 100644,000000..100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/reconciliation/ReconcileContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/reconciliation/ReconcileContainerEventHandler.java
@@@ -1,92 -1,0 +1,92 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.reconciliation;
+
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.reconciliation.ReconciliationEligibilityHandler.EligibilityResult;
+import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.apache.hadoop.ozone.protocol.commands.ReconcileContainerCommand;
+import org.apache.ratis.protocol.exceptions.NotLeaderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * When a reconcile container event is fired, this class will check if the container is eligible for reconciliation,
+ * and if so, send the reconcile request to all datanodes with a replica of that container.
+ */
+public class ReconcileContainerEventHandler implements EventHandler<ContainerID> {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ReconcileContainerEventHandler.class);
+
+ private final ContainerManager containerManager;
+ private final SCMContext scmContext;
+
+ public ReconcileContainerEventHandler(ContainerManager containerManager, SCMContext scmContext) {
+ this.containerManager = containerManager;
+ this.scmContext = scmContext;
+ }
+
+ @Override
+ public void onMessage(ContainerID containerID, EventPublisher publisher) {
+ if (!scmContext.isLeader()) {
+ LOG.info("Skip reconciling container {} since current SCM is not leader.", containerID);
+ return;
+ }
+
+ EligibilityResult result = ReconciliationEligibilityHandler.isEligibleForReconciliation(containerID,
+ containerManager);
+ if (!result.isOk()) {
+ LOG.error("{}", result);
+ return;
+ }
+
+ try {
+ // TODO HDDS-10714 restriction peer and target nodes based on node status.
+ Set<DatanodeDetails> allReplicaNodes = containerManager.getContainerReplicas(containerID)
+ .stream()
+ .map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toSet());
+
+ LOG.info("Reconcile container event triggered for container {} with peers {}", containerID, allReplicaNodes);
+
+ for (DatanodeDetails replica : allReplicaNodes) {
+ Set<DatanodeDetails> otherReplicas = allReplicaNodes.stream()
+ .filter(other -> !other.equals(replica))
+ .collect(Collectors.toSet());
+ ReconcileContainerCommand command = new ReconcileContainerCommand(containerID.getId(), otherReplicas);
+ command.setTerm(scmContext.getTermOfLeader());
- publisher.fireEvent(DATANODE_COMMAND, new CommandForDatanode<>(replica.getUuid(), command));
++ publisher.fireEvent(DATANODE_COMMAND, new CommandForDatanode<>(replica, command));
+ }
+ } catch (ContainerNotFoundException ex) {
+ LOG.error("Failed to start reconciliation for container {}. Container not found.", containerID);
+ } catch (NotLeaderException nle) {
+ LOG.info("Skip reconciling container {} since current SCM is not leader.", containerID);
+ }
+ }
+}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java
index fd21e0b102,d2777e50e3..4363cb8830
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java
@@@ -52,9 -69,9 +52,8 @@@ import org.apache.hadoop.ozone.containe
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ozone.test.GenericTestUtils.LogCapturer;
import org.apache.ozone.test.LambdaTestUtils;
import org.junit.jupiter.api.AfterAll;
- import org.junit.jupiter.api.Timeout;
/**
* This class tests the data scanner functionality.
diff --cc hadoop-ozone/recon-codegen/src/main/java/org/apache/ozone/recon/schema/ContainerSchemaDefinition.java
index c1d3197cb1,8a28d0eee0..5919028a56
--- a/hadoop-ozone/recon-codegen/src/main/java/org/apache/ozone/recon/schema/ContainerSchemaDefinition.java
+++ b/hadoop-ozone/recon-codegen/src/main/java/org/apache/ozone/recon/schema/ContainerSchemaDefinition.java
@@@ -105,4 -90,18 +90,19 @@@ public class ContainerSchemaDefinition
public DataSource getDataSource() {
return dataSource;
}
+
+ /**
+ * ENUM describing the allowed container states which can be stored in the
+ * unhealthy containers table.
+ */
+ public enum UnHealthyContainerStates {
+ MISSING,
+ EMPTY_MISSING,
+ UNDER_REPLICATED,
+ OVER_REPLICATED,
+ MIS_REPLICATED,
+ ALL_REPLICAS_BAD,
- NEGATIVE_SIZE // Added new state to track containers with negative sizes
++ NEGATIVE_SIZE, // Added new state to track containers with negative sizes
++ REPLICA_MISMATCH
+ }
}
diff --cc hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UnhealthyContainerReplicaMismatchAction.java
index 7283858e80,0000000000..d50b16557e
mode 100644,000000..100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UnhealthyContainerReplicaMismatchAction.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UnhealthyContainerReplicaMismatchAction.java
@@@ -1,99 -1,0 +1,98 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.upgrade;
+
+import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.UNHEALTHY_CONTAINER_REPLICA_MISMATCH;
+import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.FINALIZE;
+import static org.apache.ozone.recon.schema.ContainerSchemaDefinition.UNHEALTHY_CONTAINERS_TABLE_NAME;
+import static org.apache.ozone.recon.schema.SqlDbUtils.TABLE_EXISTS_CHECK;
+import static org.jooq.impl.DSL.field;
+import static org.jooq.impl.DSL.name;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Arrays;
+import javax.sql.DataSource;
- import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
+import org.apache.ozone.recon.schema.ContainerSchemaDefinition;
+import org.jooq.DSLContext;
+import org.jooq.impl.DSL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Upgrade action for handling the addition of a new unhealthy container state in Recon, which will be for containers,
+ * that have replicas with different data checksums.
+ */
+@UpgradeActionRecon(feature = UNHEALTHY_CONTAINER_REPLICA_MISMATCH, type = FINALIZE)
+public class UnhealthyContainerReplicaMismatchAction implements ReconUpgradeAction {
- private static final Logger LOG = LoggerFactory.getLogger(InitialConstraintUpgradeAction.class);
++ private static final Logger LOG = LoggerFactory.getLogger(UnhealthyContainerReplicaMismatchAction.class);
+ private DataSource dataSource;
+ private DSLContext dslContext;
+
+ @Override
- public void execute(ReconStorageContainerManagerFacade scmFacade) throws Exception {
- this.dataSource = scmFacade.getDataSource();
++ public void execute(DataSource source) throws Exception {
++ this.dataSource = source;
+ try (Connection conn = dataSource.getConnection()) {
+ if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) {
+ return;
+ }
+ dslContext = DSL.using(conn);
+ // Drop the existing constraint
+ dropConstraint();
+ // Add the updated constraint with all enum states
+ addUpdatedConstraint();
+ } catch (SQLException e) {
+ throw new SQLException("Failed to execute UnhealthyContainerReplicaMismatchAction", e);
+ }
+ }
+
+ /**
+ * Drops the existing constraint from the UNHEALTHY_CONTAINERS table.
+ */
+ private void dropConstraint() {
+ String constraintName = UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1";
+ dslContext.alterTable(UNHEALTHY_CONTAINERS_TABLE_NAME)
+ .dropConstraint(constraintName)
+ .execute();
+ LOG.debug("Dropped the existing constraint: {}", constraintName);
+ }
+
+ /**
+ * Adds the updated constraint directly within this class.
+ */
+ private void addUpdatedConstraint() {
+ String[] enumStates = Arrays
+ .stream(ContainerSchemaDefinition.UnHealthyContainerStates.values())
+ .map(Enum::name)
+ .toArray(String[]::new);
+
+ dslContext.alterTable(ContainerSchemaDefinition.UNHEALTHY_CONTAINERS_TABLE_NAME)
+ .add(DSL.constraint(ContainerSchemaDefinition.UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1")
+ .check(field(name("container_state"))
+ .in(enumStates)))
+ .execute();
+
+ LOG.info("Added the updated constraint to the UNHEALTHY_CONTAINERS table for enum state values: {}",
+ Arrays.toString(enumStates));
+ }
+
+ @Override
+ public UpgradeActionType getType() {
+ return FINALIZE;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org