You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by as...@apache.org on 2025/05/16 16:12:08 UTC
(ozone) 01/04: Merge branch 'master' of https://github.com/apache/ozone into HDDS-10239-container-reconciliation
This is an automated email from the ASF dual-hosted git repository.
aswinshakil pushed a commit to branch HDDS-10239-container-reconciliation
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 989746a0de358267be31ce81824046766b49bb7c
Merge: 8768d04816 5c5db8e9bc
Author: Aswin Shakil Balasubramanian <as...@gmail.com>
AuthorDate: Fri Apr 25 17:02:35 2025 +0530
Merge branch 'master' of https://github.com/apache/ozone into HDDS-10239-container-reconciliation
Commits: 318
68452d0573 HDDS-12893. addendum: cp: warning: behavior of -n is non-portable and may change in future; use --update=none instead (#8334)
6852e3eb93 HDDS-12861. Add AvoidStringBufferField PMD rule (#8312)
c21ec5b645 HDDS-12880. Move field declarations to start of class in hdds-server-framework (#8310)
56b5b02317 HDDS-12707. Recon - In-memory extraction of om tarball from network input stream (#8212)
64dbf44d3c HDDS-12893. cp: warning: behavior of -n is non-portable and may change in future; use --update=none instead (#8328)
250d3c1734 HDDS-12891. OMKeyAclRequestWithFSO is incorrectly setting full path as key name. (#8326)
f6597d1b2d HDDS-12888. Add negative test cases for FS operations on OBS buckets (#8323)
cdab4eb42d HDDS-12173. Follow RocksDB basic tuning guide (#8206)
6f25b439b1 HDDS-12347. Fix spotbugs warnings in integration tests (#8298)
e83569f80b HDDS-12816. Ozone debug replicas chunk-info has incorrect or None in FileLocations (#8289)
621f018134 HDDS-12624. Fix pipeline limit check to prevent incorrect pipeline creation (#8216)
dd8950bd96 HDDS-12340. Remove unnecessary findbugs exclusions (#8309)
0d88da0b71 HDDS-12723. Handle Volume Db failure in volume scanner (#8256)
434a5d539a HDDS-12823. SnapshotDiffReportOzone#fromProtobuf empty token handling (#8280)
7c23cbdf66 HDDS-12504. Replace calls to deprecated RandomStringUtils methods (#8306)
3ee57e6052 HDDS-12813. Replace calls to deprecated RandomUtils methods (#8302)
84b2ff6d66 HDDS-12876. Bump awssdk to 2.31.25 (#8304)
db9076e11e HDDS-12867. Replace hard-coded namespace URL with constant S3_XML_NAMESPACE (#8299)
97b35eeddc HDDS-12874. Bump picocli to 4.7.7 (#8303)
3eed6a6f37 HDDS-12847. Use DatanodeID instead of DatanodeDetails.getUuidString (#8293)
205cb2cc4d HDDS-12846. Log DatanodeDetails instead of DatanodeDetails.getUuidString (#8295)
213c2fe6a0 HDDS-11734. Bump maven-compiler-plugin to 3.14.0 (#8301)
889f3b7de8 HDDS-10284. Move GenericTestUtils#getTempPath to MiniOzoneCluster (#8300)
e613598768 HDDS-12113. Move HAProxy test to HA environment (#8271)
d8a391558f HDDS-12413. Move field declarations to start of class in hdds-container-service (#7968)
5c91b44ad1 HDDS-8802. Added pagination support for ListSnapshotDiff jobs (#8124)
e9e149336b HDDS-12704. Add missing audit logs for SCM client operations (#8251)
e8af186de1 HDDS-12152. Stop testing with Hadoop 3.1.2 (#7773)
f6a4a48b0b HDDS-12840. Avoid long containerId in KeyValueContainer (#8291)
3c9e1465e9 HDDS-12350. Reduce duplication between OmBucketReadWrite tests (#8241)
93613c1cae HDDS-12734. Enable native lib in CI checks (#8190)
3d4b5fdf55 HDDS-6631. Fix typos in output/exception messages (#8294)
978dd717f8 HDDS-12806. Replace commons-logging with jcl-over-slf4j (#8265)
ef8e0da2c1 HDDS-12524. Reuse TestDataUtil.createKey in more tests (#8270)
b91e8e732f HDDS-12518. Auto-compact tables which can tend to be large in size at intervals (#8260)
55f6924a1f HDDS-12845. Reuse TestDataUtil.createKey in OzoneRpcClientTests (#8274)
8a4deeb1cf HDDS-12827. Move out NodeStateMap.Entry and ContainerMap.Entry (#8273)
62df3069f8 HDDS-12493. Move container upgrade under repair (#8205)
b33ed23605 HDDS-12412. Make hdds-common compliant with FieldDeclarationsShouldBeAtStartOfClass (#7967)
a95d3389e0 HDDS-12740. Use DatanodeID in HddsTestUtils (#8281)
f26e2f06a3 HDDS-12771. xcompat fails if run in itself due to undefined OZONE_CURRENT_VERSION (#8236)
c7117dcc17 HDDS-11974. Split Container Safemode Rule into Ratis & EC Container Safemode Rules (#7951)
bb2c63c502 HDDS-12803. OmKeyInfo#isKeyInfoSame should handle object tags (#8259)
a27c1cf3a6 HDDS-12837. Bump vite to 4.5.13 (#8283)
9b48b6e8c1 HDDS-12838. Bump awssdk to 2.31.21 (#8267)
3d8644e172 HDDS-12830. Add RocksDatabaseException. (#8277)
9e4da5bac2 HDDS-12821. Update Build from Source user doc. (#8262)
5292ac57d1 HDDS-12836. Bump junit to 5.12.2 (#8269)
e6b9d5a123 HDDS-12825. ReconIncrementalContainerReportHandler is not synchronized on datanode. (#8272)
a92fe59ec4 HDDS-12581. Multi-threaded Log File Parsing with Batch Updates to DB (#8254)
eed5924251 HDDS-12770. Use ContainerID instead of Long in CONTAINER_IDS_TABLE. (#8247)
7dfd8c1a78 HDDS-12060. Replace System.currentTimeMillis() with Time.monotonicNow() for duration calculation (#8096)
a21e362b6a HDDS-12547. Container creation and import use the same VolumeChoosingPolicy (#8090)
22734a91f2 HDDS-12461. Bump Ranger to 2.6.0 (#8120)
0d6231c2b3 HDDS-12801. SCM should remove pipeline before DN. (#8261)
fdebdacc18 HDDS-12761. Add missing network configuration properties in ozone-default.xml (#8257)
7062685609 HDDS-12805. Use slf4j for HTTP request log (#8255)
7ff8ad7a13 HDDS-12755. Redundant declaration in TestHadoopNestedDirGenerator#spanCheck() (#8218)
fbb8706a7b HDDS-12145. Remove unused config hdds.container.ratis.replication.level (#8252)
691383526a HDDS-12519. Generate auto-complete script for Ozone commands (#8030)
25dda2d954 HDDS-12408. Create mixin for ContainerID list parameters (#7970)
f22f32e05d HDDS-12764. NodeDecommissionManager#monitorInterval should get with ms unit from config (#8250)
dc9952e446 HDDS-12746. Reduce visibility of Logger members (#8210)
102ae3fd26 HDDS-8660. Notify ReplicationManager when nodes go dead or out of service (#7997)
a1345f9fdd HDDS-12368. Seek to correct start key in KeyManagerImpl#getTableEntries (#7925)
1ebf2652b1 HDDS-12580. Set up Temporary DB for Storing Container Log Information (#8072)
189fbdbe4c HDDS-12463. Add perf metrics in SCM for allocateBlock and openPipeline (#8111)
fa8afa339c HDDS-12741. Recon UI should show full path from ContainerKeys API response (#8200)
a0387afc88 HDDS-12800. Bump aspectj-maven-plugin to 1.14.1 (#8230)
5733c5556e HDDS-11799. Remove config hdds.scm.safemode.pipeline-availability.check (#8095)
c962b98cef HDDS-12404. Grafana dashboard for snapshot metrics (#7954)
bf20540044 HDDS-12458. Refactor DataNodeSafeModeRule to use NodeManager (#7998)
25b7102e30 HDDS-12775. flaky-test-check builds the workflow branch (#8237)
2a4de14eb8 HDDS-12772. Configure initial heartbeat and first election time for quicker MiniOzoneCluster startup (#8235)
0f5b59089b HDDS-12691. Calculation of committed space in Datanode seems incorrect (#8228)
6d0a8306ab HDDS-12721. Log more details about volumes (#8181)
1bee369bb2 HDDS-12525. Replace some TestHelper#createKey usage with TestDataUtil#createKey (#8233)
5834fcf7c8 HDDS-10091. TestOzoneDebugShell fails with FILE_SYSTEM_OPTIMIZED bucket layout (#8227)
38229439fa HDDS-12768. Bump vite to 4.5.12 (#8234)
b6dac4ae33 HDDS-12766. Bump awssdk to 2.31.16 (#8232)
0ef1f0208e HDDS-12767. Bump jacoco to 0.8.13 (#8231)
2bbeaafe1f HDDS-12700. Upgrade test overwrites previous result (#8229)
0a558c15e0 HDDS-12758. Error in OmUtils.normalizeKey for key name starting with `//` (#8225)
fee8817193 HDDS-12759. Bump vitest to 1.6.1 (#7828)
1f716eae7d HDDS-12699. Bump vite to 4.5.11 (#8224)
abc1e02834 HDDS-12757. Duplicated declaration of dnsInterface in HddsUtils (#8222)
3fb57c1695 HDDS-12037. Removing unit from quota namespace (#8148)
ebcece4272 HDDS-11038. Add Helm Chart to the Ozone on Kubernetes doc (#8220)
bba8a67831 HDDS-12750. Move StorageTypeProto from ScmServerDatanodeHeartbeatProtocol.proto to hdds.proto (#8208)
bc19a4c18a HDDS-12711. Limit number of excluded SST files logged at info level (#8186)
6bcebe8a0c HDDS-4517. Remove leftover references to RaftServerImpl (#8223)
868a2376fe HDDS-12756. Speed up TestReconfigShell and TestOzoneDebugShell (#8219)
40d02b9165 HDDS-12639. Add info for TimeoutException (#8113)
5f74da38ff HDDS-12760. Mark TestContainerReplication#testImportedContainerIsClosed as flaky
ed0111582f HDDS-9241. Document S3 Gateway REST Secret Revoke command (#8221)
479f6d949f HDDS-12690. Remove om.fs.snapshot.max.limit from RDBStore and OmMetadataManagerImpl (#8195)
0f52a34d0c HDDS-12559. Implement Bulk Ozone Locks for taking locks on multiple snapshots (#8052)
de73c00a8d HDDS-12738. Refactor AbstractContainerReportHandler and its subclasses. (#8207)
30e4aa44d8 HDDS-12748. Remove unused config ozone.manager.db.checkpoint.transfer.bandwidthPerSec (#8213)
693548b037 HDDS-12676. Prefer minFreeSpace if minFreeSpacePercent is also defined (#8180)
092fe403c4 HDDS-12751. native build fails with CMake 4 (#8215)
1433d0c047 HDDS-1480. Prefer resolved datanode ip address over persisted ip address (#7495)
af5301e380 HDDS-12233. Atomically import a container (#7934)
2475949d1b HDDS-11107. Remove unnecessary run_test_scripts call in upgrade tests (#8201)
20a13da5ef HDDS-12615. Failure of any OM task during bootstrapping of Recon needs to be handled (#8098)
7fb92b0144 HDDS-12717. Combine nodeMap and nodeToContainer in NodeStateMap. (#8199)
fc6776db33 HDDS-12739. Replace the getNodeByUuid methods in NodeManager. (#8202)
0d40030469 HDDS-12698. Unused FailureService in MiniOzoneChaosCluster (#8197)
400835fd9e HDDS-12736. Bump hadoop-thirdparty to 1.4.0 (#8193)
a540684346 HDDS-12720. Use DatanodeID in SimpleMockNodeManager (#8198)
604576a0c9 HDDS-12735. Unused rocksDBConfiguration variable in `OmMetadataManagerImpl#start` (#8196)
64318f78a7 HDDS-12498. Allow limiting flaky-test-check to specific submodule (#8194)
3e7b5eb443 HDDS-12719. Remove the setContainers(..) method from NodeManager. (#8191)
78e2e73158 HDDS-12724. hdds-rocks-native build fails if JAVA_HOME not set (#8183)
fbc696cc44 HDDS-12594. Optimize replica checksum verifier (#8151)
4411e35d53 HDDS-12733. Bump axios to 0.30.0 (#8189)
31aabc1efd HDDS-12732. Bump awssdk to 2.31.11 (#8188)
d6e2509771 HDDS-12731. Bump restrict-imports-enforcer-rule to 2.6.1 (#8187)
4213307637 HDDS-12648. Fix checkLeaderStatus in removeOMFromRatisRing and addOMToRatisRing (#8142)
e4e95470d3 HDDS-12709. Intermittent failure in Balancer acceptance test (#8182)
c1d5b4f390 HDDS-12718. Use NodeStateMap in MockNodeManager instead of test-specific Node2ContainerMap (#8179)
65a9d6ce31 HDDS-12587. Detect test class module in flaky-test-check (#8162)
aac938306b HDDS-12529. Clean up code in AbstractContainerReportHandler. (#8033)
68913156a7 HDDS-12603. Fix ContainerTable keys re-rendering issue (#8163)
2a1a6bf124 HDDS-12670. Improve encapsulation of volume spare space check (#8167)
05b6eb3a61 HDDS-12696. Replace link to Hadoop with Ozone in httpfs site.xml (#8177)
7e87a8a890 HDDS-11879. MiniKMS fails with ClassNotFoundException: com.sun.jersey....ServletContainer (#8158)
841d297104 HDDS-12592. Remove find missing padding check in favor of of block metadata check (#8145)
ce003e898b HDDS-12702. Move checknative under ozone debug (#8170)
6ee9c2bbb3 HDDS-8007. Add more detailed stages for SnapDiff job progress tracking (#8010)
3ebe5e7cd8 HDDS-12660. Allow --verbose option of GenericCli at leaf subcommands (#8166)
c8b3ccb01b HDDS-12528. Create new module for S3 integration tests (#8168)
435dee9896 HDDS-12703. Close pipeline command should display error on failure (#8169)
deb8e7c472 HDDS-12705. Replace Whitebox with HddsWhiteboxTestUtils (#8172)
0d8aecc01e HDDS-12327. HDDS-12668. Fix HSync upgrade test failure in non-HA upgrade test (#8171)
bef4beed82 HDDS-12239. (addendum) Volume should not be marked as unhealthy when disk full - fix compile error
e3e47ea41e HDDS-12235. Reserve space on DN during container import operation. (#7981)
4ffae705c7 HDDS-12239. Volume should not be marked as unhealthy when disk full (#7830)
0781cbede3 HDDS-12694. Disable TestMiniChaosOzoneCluster after fixing init and shutdown (#8159)
92aa71debf HDDS-12687. Avoid ambiguity in URI descriptions (#8155)
536701649e HDDS-12679. Merge VolumeInfo into StorageVolume (#8147)
2cf6d59e1e HDDS-12553. ozone admin container list should output real JSON array (#8050)
63fcb271a7 HDDS-12465. Intermittent failure in TestOzoneFileSystemMetrics (#8130)
bab26a43fa HDDS-12684. Update NOTICE and LICENSE file (#8160)
857cb768b7 HDDS-12327. Restore non-HA (to HA) upgrade test (#7880)
587e9ff1be HDDS-12320. Collect OM performance metrics for FSO key delete (#7883)
560d017bc7 HDDS-12500. Do not skip JUnit tests in post-commit runs (#8024)
273a627692 HDDS-12686. Remove output of OzoneAddress in --verbose mode CLI (#8153)
4d3d834c60 HDDS-12650. Added logs to SnapshotDeletingService to indicate skipped snapshots. (#8123)
752473628d Revert "HDDS-12528. Create new module for S3 integration tests (#8152)"
d44ebf564c HDDS-12528. Create new module for S3 integration tests (#8152)
3739b0597b HDDS-12604. Reduce duplication in TestContainerStateMachine (#8104)
a1616ae8a2 HDDS-12485. Repair tool should only print user warning for offline commands (#8140)
de5c0a385e HDDS-12668. HSync upgrade test failure (#8137)
bee10f0731 HDDS-12671. Include .editorconfig and .run in source tarball (#8139)
8e7d370ebf HDDS-12666. Remove -SNAPSHOT from OZONE_CURRENT_VERSION in upgrade test (#8136)
4e0a76464c HDDS-12486. Warmup KMS encrypted keys when OM starts (#8081)
09109978ac HDDS-12636. Reduce code duplication for tarball creation (#8121)
cbafa02918 HDDS-12642. ACL test assertions depend on JSON element order (#8143)
8691adfc44 HDDS-12622. Refactor minFreeSpace calculation (#8119)
1b0e912f53 HDDS-12661. Standardize Maven module names (#8129)
ebf5cc662b HDDS-12662. Rename upgrade callback directory 1.5.0 to 2.0.0 (#8131)
828b2d116b HDDS-12667. Bump awssdk to 2.31.6 (#8134)
2596b3be08 HDDS-12665. Bump zstd-jni to 1.5.7-2 (#8133)
0e079832ae HDDS-12664. Bump copy-rename-maven-plugin to 1.0.1 (#8132)
31735012d1 HDDS-12462. Use exclude rules for defining shaded filesystem jar contents (#8008)
ee6201beb4 HDDS-12341. Share cluster in client tests (#8126)
bd579b355f HDDS-12646. Improve OM decommission check (#8122)
482024e6bc HDDS-12569. Extract MiniOzoneCluster to separate module (#8067)
7a3ad16568 HDDS-12641. Move Lease to hdds-server-framework (#8128)
e805c15c74 HDDS-12310. Online repair command to perform compaction on om.db (#7957)
7b9c15276a HDDS-12473. Trim duplicate space in proto message definition (#8005)
918bb9809c HDDS-12626. Move the compare method in NodeStatus to ECPipelineProvider. (#8116)
bf5a2f2209 HDDS-12644. Create factory method for OzoneAcl (#8115)
a646b85b82 HDDS-12557. Add progress indicator for checkpoint tarball in leader OM (#8085)
9e23d5989b HDDS-12627. NodeStateMap may handle opStateExpiryEpochSeconds incorrectly. (#8117)
d19f3bc992 HDDS-12640. Move GrpcMetrics to hdds-server-framework (#8114)
1c0b445a5d HDDS-12426. SCM replication should check double of container size. (#8080)
157cca451b HDDS-12358. Intermittent failure in compatibility acceptance test (#8012)
9b25d029b5 HDDS-12549. refactor ratis request to common place (#8059)
3349248d9d HDDS-12588. Recon Containers page shows number of blocks, not keys (#8074)
693d0f483e HDDS-12589. Fix Incorrect FSO Key Listing for Container-to-Key Mapping. (#8109)
17dbc4f134 HDDS-12585. Recon ContainerHealthTask ConstraintViolationException error handling. (#8070)
53c0a320a3 HDDS-12620. Fix OM Mismatch Deleted Container API (#8102)
45c900d5d9 HDDS-12619. Optimize Recon OM Container Mismatch API (#8101)
57fda0c327 HDDS-12633. KEY_NOT_FOUND in OzoneRpcClientTests for LEGACY bucket with enableFileSystemPaths=true (#8108)
23b0505d2e Revert "HDDS-12630. Enable GitHub Discussions in asf.yml (#8107)"
b402b7c1ad HDDS-12621. Change NodeStatus to value-based. (#8105)
20aeda57f3 HDDS-12601. Unknown tarball cleanup for Recon OM DB snapshot. (#8084)
93cd5aa442 HDDS-12630. Enable GitHub Discussions in asf.yml (#8107)
61a36f6312 HDDS-12373. Use File.getUsableSpace() instead of File.getFreeSpace() to calcuate usedSpace in DedicatedDiskSpaceUsage (#7927)
953e718872 HDDS-12565. Treat volumeFreeSpaceToSpare as reserved space (#8086)
8489cc86b5 HDDS-11735. Update ozone-default.xml for volume choosing policy (#8103)
13c5be850b HDDS-12577. [Ozone 2.0] Update master branch version number.
1a9d9f77f7 HDDS-12446. Add a Grafana dashboard for low level RocksDB operations. (#7992)
c9990cacaa HDDS-12576. [Ozone 2.0] Update proto.lock files (#8064)
b68b94cc08 HDDS-12617. Use DatanodeID as keys in NodeStateMap. (#8100)
b71d4086b1 Revert "HDDS-12589. Fix Incorrect FSO Key Listing for Container-to-Key Mapping. (#8078)"
68e3842616 HDDS-12589. Fix Incorrect FSO Key Listing for Container-to-Key Mapping. (#8078)
c8e77f8f06 HDDS-12602. Intermittent failure in TestContainerStateMachine.testWriteFailure (#8099)
08ac32dc23 HDDS-12608. Race condition in datanode version file creation (#8093)
e5ef35d815 HDDS-12582. TypedTable support using different codec (#8073)
dc47897721 HDDS-12611. Snapshot creation is removing extra keys from AOS's DB (#8094)
ff3ef5112e HDDS-12551. Replace dnsToUuidMap with dnsToDnIdMap in SCMNodeManager. (#8087)
28520a7e01 HDDS-12614. Configurable java version in flaky-test-check with default to 21 (#8097)
434d5bd248 HDDS-12616. Bump junit to 5.12.1 (#8088)
699ee88683 HDDS-12610. Bump awssdk to 2.31.1 (#8089)
0a85f9b3ce HDDS-12602. Mark TestContainerStateMachineLeader/Follower as flaky
03c80f8d9b HDDS-12591. Include ContainerInfo in ContainerAttribute. (#8083)
49a2c853d2 HDDS-12097. Enhance Container Key Mapper for Faster Processing. (#8002)
5b7f96c21d HDDS-12303. Move ozone.om.user.max.volume into OmConfig (#8082)
afc40430c0 HDDS-12566. Handle Over replication of Quasi Closed Stuck containers (#8061)
eb96ff4892 HDDS-12590. Used db name as the threadNamePrefix. (#8076)
cf5bad75cf HDDS-12539. Enable some String-related rules in PMD (#8047)
d26f711e27 HDDS-12533. Offline repair command for generic rocksDB compaction (#8039)
93563e99d6 HDDS-12573. Pipeline#toString should separate ReplicaIndex from next node UUID. (#8063)
6e40831f27 HDDS-12572. Remove the ContainerID parameter when it has ContainerReplica. (#8075)
786da39cf5 HDDS-12057. Implement command ozone debug replicas verify checksums (#7748)
66bc7eaa3d HDDS-12535. Intermittent failure in TestContainerReportHandling (#8060)
87a674c875 HDDS-12555. Combine containerMap and replicaMap in ContainerStateMap. (#8057)
f3689b692c HDDS-12458. Show safemode rules status irrespective of whether SCM is in safe mode in verbose mode. (#8049)
7d8c771aad HDDS-12469. Mark statemachine unhealthy for write operation timeout. (#8022)
9aa41fdbaf HDDS-12574. Add script to find modules by test classes (#8062)
5e28acb71d HDDS-12537. Selective checks: skip tests for PMD ruleset change (#8040)
9a3433b0d2 HDDS-12520. Move auditparser under debug (#8041)
0335385e5e HDDS-12420. Move FinalizeUpgradeCommandUtil to hdds-common. (#8023)
75bbba30a8 HDDS-12552. Fix raw use of generic class SCMCommand (#8048)
d71aadf0f1 HDDS-12527. Separate S3 Gateway from MiniOzoneCluster (#8058)
de8cf168ad HDDS-12483. Quasi Closed Stuck should have 2 replicas of each origin (#8014)
57a139e692 HDDS-11576. Create a separate S3 client factory (#8051)
e8a4668074 HDDS-12383. Fix spotbugs warnings in hdds-common and httpfsgateway (#8046)
0dab553d70 HDDS-12541. Change ContainerID to value-based (#8044)
1202f6df15 HDDS-12532. Support only Enum in ContainerAttribute. (#8036)
a2ad1e334d HDDS-12536. Move InMemoryTestTable to test (#8043)
7164c76ff3 HDDS-12488. S3G should handle the signature calculation with trailers (#8020)
c8c6d0e5a7 HDDS-12534. Remove drop_column_family command from ozone debug ldb (#8038)
9a8321ef78 HDDS-12535. Mark TestContainerReportHandling as flaky
ddd89fb2f3 HDDS-12543. Remove duplicate license information (#8045)
08e2c0a018 HDDS-12531. Use AtomicFileOutputStream to write YAML files. (#8035)
1bd8d8f56f HDDS-11813. Reduce duplication in CI workflow (#7497)
250bd5f317 HDDS-12450. Enable SimplifiableTestAssertion check in PMD (#8032)
96273ae699 HDDS-12489. Intermittent timeout in TestSCMContainerManagerMetrics.testReportProcessingMetrics (#8021)
231592705c HDDS-12476. Add TestDataUtil#createKey variant with small random content (#8028)
d95ca4c44e HDDS-12421. ContainerReportHandler should not make the call to delete replicas (#7976)
26c859cc3a HDDS-12204. Improve failover logging (#7867)
efbf79caf9 HDDS-12236. ContainerStateMachine should not apply or write future transactions in the event of failure (#7862)
3f88dbee51 HDDS-12377. Improve error handling of OM background tasks processing in case of abrupt crash of Recon. (#7960)
72da3a6aa3 HDDS-12477. Do not force RATIS/ONE replication in TestDataUtil#createKey (#8017)
a428b15acb HDDS-12496. Use TextFormat#shortDebugString to flatten proto message in SCMDatanodeProtocolServer. (#8019)
ecd2de095e HDDS-12409. Log an error before increasing the sequence id of a CLOSED container in SCM (#7964)
978e4a7eb6 HDDS-12168. Create new Grafana panel to display cluster growth rate (#7978)
9ab7c70380 HDDS-12456. Avoid FileInputStream and FileOutputStream (#8015)
9d41cd78a5 HDDS-12474. Add latency metrics of deletion services to grafana dashboard (#8007)
2b48e8c6ec HDDS-12354. Move Storage and UpgradeFinalizer to hdds-server-framework (#7973)
ed737b3a38 HDDS-12428. Avoid force closing OPEN/CLOSING replica of a CLOSED Container (#7985)
d1e8b90cbb HDDS-12210. Use correct BootstrapStateHandler.Lock in SnapshotDeletingService (#7991)
b769a26481 HDDS-12295. Allow updating OM default replication config for tests (#7974)
dd74eee7fc HDDS-12430. Document in ozone-default.xml the config keys moved from DFSConfigKeysLegacy (#7987)
34041caede HDDS-12466. Set default commit message to PR title (#8013)
1ea073560f HDDS-12467. Enable new asf.yaml parser (#8011)
e87b8dbb2f HDDS-12417. Reduce duplication of createKey variants in TestDataUtil (#7999)
86d2027e32 HDDS-12193. Provide option to disable RDBStoreMetrics for Snapshotted DB (#7982)
7c1d201b2e HDDS-12451. Create factory for MultiTenantAccessController (#7996)
c2a934ceb0 HDDS-12470. Revert workaround added by HDDS-8715 to preserve thread name. (#8004)
b8c93ccf46 HDDS-12376. (addendum: fix findbugs) Remove scmRatisEnabled from ScmInfo. (#7931)
4005a104db HDDS-12376. (addendum: fix pmd) Remove scmRatisEnabled from ScmInfo. (#7931)
abfa3becfa HDDS-12442. Add latency metrics for OM deletion services (#7986)
14db15cd67 HDDS-12376. Remove scmRatisEnabled from ScmInfo. (#7931)
7d31d9e522 HDDS-12410. Add detailed block info for ALLOCATE_BLOCK audit log (#7965)
a8f0ff3d7b HDDS-12460. (addendum) Move hdds-test-utils code to src/test
4c28c7f62d HDDS-12460. Move hdds-test-utils code to src/test (#8000)
bb16f66e22 HDDS-12448. Avoid using Jackson1 (#7994)
be0e1e6a86 HDDS-12445. Remove unused code from ContainerStateMap. (#7990)
a3c9c0e040 HDDS-12443. Intermittent failure in TestContainerBalancerSubCommand (#7989)
83fd8d7a31 HDDS-12449. Enable UseCollectionIsEmpty check in PMD (#7995)
d18da13da4 HDDS-12452. Bump slf4j to 2.0.17 (#7993)
6a9e8b148f HDDS-12348. Reuse `TestDataUtil.createKey` method (#7971)
384d774254 HDDS-12416. Enable UnusedPrivateField check in PMD (#7975)
39d7da3713 HDDS-12424. Allow config key to include config group prefix (#7979)
ba0939bc67 HDDS-12156. Add container health task metrics in Recon. (#7786)
d34aee40c5 HDDS-12172. Rename Java constants of DFSConfigKeysLegacy keys (#7922)
59aaa5cdb0 HDDS-12418. Remove healthyReplicaCountAdapter from RatisContainerReplicaCount (#7972)
232e780902 HDDS-12150. Abnormal container states should not crash the SCM ContainerReportHandler thread (#7882)
a708ea4fdd HDDS-12351. Move SCMHAUtils and ServerUtils to hdds-server-framework (#7961)
052bd2deef HDDS-12198. Exclude Recon generated code in coverage (#7962)
ba44e12da3 HDDS-12382. Fix other spotbugs warnings (#7969)
e2bf5998c1 HDDS-12345. Share cluster in filesystem tests (#7959)
f9f1c80dc6 HDDS-12411. Make hdds-client compliant with FieldDeclarationsShouldBeAtStartOfClass (#7966)
46f4986482 HDDS-11512. Create Grafana dashboard for tracking system wide deletion (#7813)
f7e9aedf49 HDDS-12396. Enable UnusedPrivateMethod check in PMD (#7956)
619f524517 HDDS-1234. Fix spotbugs warnings in ozone-manager (#7963)
1d64b37c64 HDDS-12380. Fix spotbugs warnings in hdds-container-service (#7958)
5c2b8f649f HDDS-12065. Checkpoint directory should be cleared on startup (#7681)
a31755a085 HDDS-12315. Speed up some Freon integration tests (#7870)
d8f3149211 HDDS-12399. Enable PMD.ForLoopCanBeForeach rule (#7952)
393211a6e8 HDDS-12062. Recon - Error handling in NSSummaryTask to avoid data inconsistencies. (#7723)
ecc240330b HDDS-9792. Add tests for Pipelines page (#7859)
e4de75e4b3 HDDS-12388. Key rewrite tests should be skipped if feature is disabled (#7953)
7b0fe618b1 HDDS-12288. (addendum) fix checkstyle
d5df7e43c7 HDDS-12288. Improve bootstrap logging to indicate progress of snapshot download. (#7861)
58a04536f7 HDDS-12381. Fix spotbugs warnings in TestHddsUtils (#7955)
1a0c2238fa HDDS-11768. Extract SCM failover proxy provider logic (#7950)
b45de0f18b HDDS-12403. Bump zstd-jni to 1.5.7-1 (#7949)
a57809e5ad HDDS-12402. Bump sqlite-jdbc to 3.49.1.0 (#7948)
221d53dc3a HDDS-12400. Bump junit to 5.12.0 (#7947)
f17abae511 HDDS-12398. Enable PMD checks for tests (#7946)
540f67b12f HDDS-12353. Move SpaceUsage implementations to hdds-server-framework (#7926)
70b93dc6a6 HDDS-10764.Tarball creation failing on leader OM node. (#7941)
6b20afcd13 HDDS-12387. Cleanup TestContainerOperations (#7940)
417ae7c62a HDDS-12365. Provide editor settings and IDEA run config in standard location (#7924)
a2c5c8eaf0 HDDS-12293. Make ozone.om.server.list.max.size reconfigurable (#7938)
e0bd2cc716 HDDS-12185. Enhance FileSizeCountTask for Faster Processing. (#7796)
589eeef871 HDDS-12371. Duplicated key scanning on multipartInfo table when listing multipart uploads (#7937)
6e766bffee HDDS-12367. Change ignorePipeline log level to DEBUG in OmKeyInfo (#7939)
c87caa577d HDDS-12226. TestSecureOzoneRpcClient tests not run due to UnknownHostException (#7827)
1513c34e2f HDDS-11883. SCM HA: Move proxy object creation code to SCMRatisServer (#7914)
8c42c026ef HDDS-12349. Speed up some HDDS integration tests (#7932)
a00787c9d1 HDDS-11532. Sort multipart uploads on ListMultipartUploads response (#7929)
Conflicts:
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java
hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java
.asf.yaml | 6 +
.editorconfig | 77 +
.github/workflows/check.yml | 289 +
.github/workflows/ci.yml | 589 +-
.github/workflows/intermittent-test-check.yml | 76 +-
.../CsiServer.xml => .run/CsiServer.run.xml | 0
.../Datanode1-ha.xml => .run/Datanode1-ha.run.xml | 0
.../Datanode1.xml => .run/Datanode1.run.xml | 0
.../Datanode2-ha.xml => .run/Datanode2-ha.run.xml | 0
.../Datanode2.xml => .run/Datanode2.run.xml | 0
.../Datanode3-ha.xml => .run/Datanode3-ha.run.xml | 0
.../Datanode3.xml => .run/Datanode3.run.xml | 0
.../FreonStandalone.run.xml | 0
.../OzoneFsShell-ha.run.xml | 0
.../OzoneFsShell.xml => .run/OzoneFsShell.run.xml | 0
.../OzoneManager-ha.run.xml | 0
.../OzoneManager.xml => .run/OzoneManager.run.xml | 0
.../OzoneManagerInit-ha.run.xml | 0
.../OzoneManagerInit.run.xml | 0
.../OzoneShell-ha.run.xml | 0
.../OzoneShell.xml => .run/OzoneShell.run.xml | 0
.../PrimordialSCM-ha.run.xml | 0
.../PrimordialSCMInit-ha.run.xml | 0
.../Recon-ha.xml => .run/Recon-ha.run.xml | 0
.../Recon.xml => .run/Recon.run.xml | 0
.../S3Gateway.xml => .run/S3Gateway.run.xml | 0
.../Scm2-ha.xml => .run/Scm2-ha.run.xml | 0
.../Scm2Bootstrap-ha.run.xml | 0
.../Scm3-ha.xml => .run/Scm3-ha.run.xml | 0
.../Scm3Bootstrap-ha.run.xml | 0
.../ScmRoles.xml => .run/ScmRoles.run.xml | 0
.../StorageContainerManager.run.xml | 0
.../StorageContainerManagerInit.run.xml | 0
CONTRIBUTING.md | 23 +-
NOTICE.txt | 2 +-
dev-support/ci/categorize_basic_checks.sh | 2 +-
dev-support/ci/find_test_class_project.bats | 239 +
dev-support/ci/find_test_class_project.sh | 98 +
dev-support/ci/selective_ci_checks.bats | 37 +-
dev-support/ci/selective_ci_checks.sh | 42 +-
dev-support/pmd/pmd-ruleset.xml | 23 +-
dev-support/pom.xml | 2 +-
hadoop-hdds/annotations/pom.xml | 4 +-
.../client/dev-support/findbugsExcludeFile.xml | 13 -
hadoop-hdds/client/pom.xml | 5 +-
.../hadoop/hdds/scm/ContainerClientMetrics.java | 1 -
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 94 +-
.../hadoop/hdds/scm/XceiverClientCreator.java | 8 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 15 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 1 -
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 49 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 8 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 4 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 12 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 12 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 3 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 1 -
.../hdds/scm/storage/ECBlockOutputStream.java | 5 +-
.../hdds/scm/storage/MultipartInputStream.java | 2 +-
.../hadoop/hdds/scm/storage/StreamBuffer.java | 1 -
.../hadoop/ozone/client/io/ECBlockInputStream.java | 3 +-
.../hdds/scm/storage/TestBlockInputStream.java | 27 +-
.../storage/TestBlockOutputStreamCorrectness.java | 2 +-
.../hadoop/hdds/scm/storage/TestBufferPool.java | 6 +-
.../hdds/scm/storage/TestChunkInputStream.java | 24 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 4 +-
.../ozone/client/io/TestECBlockInputStream.java | 3 +-
.../io/TestECBlockReconstructedInputStream.java | 3 -
.../TestECBlockReconstructedStripeInputStream.java | 6 +-
.../common/dev-support/findbugsExcludeFile.xml | 18 -
hadoop-hdds/common/pom.xml | 15 +-
.../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 73 -
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 56 +-
.../java/org/apache/hadoop/hdds/HddsIdFactory.java | 5 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 94 +-
.../java/org/apache/hadoop/hdds/JavaUtils.java | 1 -
.../java/org/apache/hadoop/hdds/StringUtils.java | 3 +-
.../apache/hadoop/hdds/cli/AbstractSubcommand.java | 5 +
.../org/apache/hadoop/hdds/cli/GenericCli.java | 7 +-
.../hadoop/hdds/cli/GenericParentCommand.java | 2 +
.../hadoop/hdds/client/DecommissionUtils.java | 1 -
.../hadoop/hdds/client/ECReplicationConfig.java | 29 +-
.../org/apache/hadoop/hdds/client/OzoneQuota.java | 46 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 67 +-
.../apache/hadoop/hdds/fs/SpaceUsageSource.java | 3 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 56 +-
.../hdds/ratis/ContainerCommandRequestMessage.java | 9 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 21 +-
.../hdds/ratis/ServerNotLeaderException.java | 17 +-
.../hadoop/hdds/ratis/conf/RatisClientConfig.java | 257 +-
.../RequestTypeDependentRetryPolicyCreator.java | 1 +
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 16 +-
.../org/apache/hadoop/hdds/scm/AddSCMRequest.java | 2 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 22 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 21 +-
.../java/org/apache/hadoop/hdds/scm/ScmInfo.java | 30 +-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 6 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 15 +-
.../hadoop/hdds/scm/container/ContainerID.java | 44 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 8 +-
.../scm/container/ReplicationManagerReport.java | 88 +-
.../common/helpers/ContainerWithPipeline.java | 1 -
.../scm/container/common/helpers/ExcludeList.java | 1 -
.../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 21 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 30 +-
.../apache/hadoop/hdds/scm/net/NetConstants.java | 9 +-
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 2 +-
.../hadoop/hdds/scm/net/NetworkTopology.java | 2 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 2 +-
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 1 +
.../org/apache/hadoop/hdds/scm/net/NodeSchema.java | 72 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 19 +-
.../hadoop/hdds/scm/net/NodeSchemaManager.java | 4 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 31 +-
.../hadoop/hdds/scm/pipeline/PipelineID.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 2 +-
.../ContainerCommandResponseBuilders.java | 4 +-
.../hadoop/hdds/scm/protocolPB/OzonePBHelper.java | 12 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 8 +-
.../hadoop/hdds/security/SecurityConstants.java | 3 +-
.../hdds/security/ssl/ReloadingX509KeyManager.java | 4 +-
.../security/ssl/ReloadingX509TrustManager.java | 11 +-
.../security/token/ShortLivedTokenIdentifier.java | 1 -
.../x509/certificate/utils/CertificateCodec.java | 7 +-
.../certificate/utils/SelfSignedCertificate.java | 2 +-
.../x509/exception/CertificateException.java | 1 +
.../org/apache/hadoop/hdds/server/JsonUtils.java | 2 +-
.../org/apache/hadoop/hdds/server/YamlUtils.java | 17 +
.../apache/hadoop/hdds/tracing/StringCodec.java | 4 +-
.../apache/hadoop/hdds/tracing/TracingUtil.java | 1 -
.../hadoop/hdds/utils/BackgroundService.java | 8 +-
.../java/org/apache/hadoop/hdds/utils/IOUtils.java | 24 +
.../org/apache/hadoop/hdds/utils/LeakTracker.java | 1 +
.../apache/hadoop/hdds/utils/db/CodecBuffer.java | 53 +-
.../hadoop/hdds/utils/db/DelegatedCodec.java | 30 +-
.../apache/hadoop/hdds/utils/db/Proto2Codec.java | 14 +-
.../apache/hadoop/hdds/utils/db/Proto3Codec.java | 14 +-
.../apache/hadoop/hdds/utils/db/ShortCodec.java | 1 -
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 18 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 24 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 20 +-
.../ozone/common/ChecksumByteBufferFactory.java | 2 +-
.../apache/hadoop/ozone/common/ChecksumCache.java | 2 +-
.../apache/hadoop/ozone/common/ChecksumData.java | 4 +-
.../ozone/common/PureJavaCrc32ByteBuffer.java | 13 +-
.../ozone/common/PureJavaCrc32CByteBuffer.java | 13 +-
.../hadoop/ozone/common/utils/BufferUtils.java | 2 +-
.../ozone/container/common/helpers/BlockData.java | 8 +-
.../container/common/helpers/ChunkInfoList.java | 8 +-
.../hadoop/ozone/lock/BootstrapStateHandler.java | 1 +
.../hadoop/ozone/upgrade/UpgradeException.java | 1 +
.../hadoop/ozone/upgrade/UpgradeFinalization.java | 175 +
.../org/apache/hadoop/ozone/util/CacheMetrics.java | 53 +-
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 2 -
.../hadoop/ozone/util/SeekableIterator.java} | 16 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 19 +-
.../hadoop/ozone/util/StringWithByteString.java | 8 +-
.../org/apache/hadoop/ozone/util/UUIDUtil.java | 10 +-
.../java/org/apache/hadoop/ozone/util/UUIDv7.java | 61 +
.../common/src/main/resources/ozone-default.xml | 327 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 49 +-
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 9 +-
.../hadoop/hdds/protocol/TestDatanodeDetails.java | 1 +
.../apache/hadoop/hdds/ratis/TestRatisHelper.java | 1 -
...TestServerNotLeaderExceptionMessageParsing.java | 12 +-
.../hdds/scm/container/TestContainerInfo.java | 21 +
.../container/TestReplicationManagerReport.java | 23 +-
.../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 2 -
.../hdds/scm/net/TestNetworkTopologyImpl.java | 26 +-
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 11 +-
.../hdds/security/x509/CertificateTestUtils.java | 4 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 1 +
.../apache/hadoop/ozone/common/TestChecksum.java | 4 +-
.../ozone/common/TestChecksumByteBuffer.java | 3 +-
.../hadoop/ozone/common/TestChecksumCache.java | 3 -
.../common/TestChecksumImplsComputeSameValues.java | 4 +-
.../ozone/container/ContainerTestHelper.java | 2 -
hadoop-hdds/config/pom.xml | 4 +-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 15 +-
.../org/apache/hadoop/hdds/conf/ConfigType.java | 12 +
.../hdds/conf/ConfigurationReflectionUtil.java | 57 +-
.../hadoop/hdds/conf/ConfigurationSource.java | 17 +-
.../hadoop/hdds/conf/ConfigurationTarget.java | 9 +-
.../org/apache/hadoop/hdds/conf/StorageSize.java | 2 +-
.../apache/hadoop/hdds/conf/TimeDurationUtil.java | 2 +-
.../hadoop/hdds/conf/ConfigurationExample.java | 14 +-
.../hadoop/hdds/conf/TestConfigFileGenerator.java | 6 +-
.../hdds/conf/TestConfigurationReflectionUtil.java | 8 +-
.../hadoop/hdds/conf/TestConfigurationSource.java | 30 +-
.../hadoop/hdds/conf/TestReconfigurableConfig.java | 2 +-
.../dev-support/findbugsExcludeFile.xml | 66 -
hadoop-hdds/container-service/pom.xml | 5 +-
.../org/apache/hadoop/hdds/scm/VersionInfo.java | 1 -
.../ozone/HddsDatanodeClientProtocolServer.java | 5 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 28 +-
.../apache/hadoop/ozone/HddsPolicyProvider.java | 14 +-
.../checksum/ContainerChecksumTreeManager.java | 10 +-
.../checksum/DNContainerOperationClient.java | 4 -
.../helpers/BlockDeletingServiceMetrics.java | 45 +-
.../common/helpers/CommandHandlerMetrics.java | 47 +-
.../container/common/helpers/ContainerMetrics.java | 6 +-
.../container/common/helpers/ContainerUtils.java | 6 +-
.../container/common/helpers/DatanodeIdYaml.java | 22 +-
.../common/helpers/DatanodeVersionFile.java | 23 +-
.../ozone/container/common/impl/ContainerData.java | 36 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../ozone/container/common/impl/ContainerSet.java | 50 +-
.../container/common/impl/HddsDispatcher.java | 24 +-
.../common/impl/OpenContainerBlockMap.java | 21 +-
.../common/impl/StorageLocationReport.java | 98 +-
.../container/common/interfaces/Container.java | 3 +
.../common/interfaces/ContainerPacker.java | 15 +
.../ozone/container/common/interfaces/Handler.java | 8 +-
.../common/report/ContainerReportPublisher.java | 3 +-
.../common/report/PipelineReportPublisher.java | 3 +-
.../container/common/report/ReportManager.java | 1 -
.../common/statemachine/DatanodeConfiguration.java | 358 +-
.../common/statemachine/DatanodeStateMachine.java | 21 +-
.../common/statemachine/EndpointStateMachine.java | 1 -
.../common/statemachine/SCMConnectionManager.java | 2 -
.../common/statemachine/StateContext.java | 20 +-
.../CloseContainerCommandHandler.java | 2 +-
.../ClosePipelineCommandHandler.java | 9 +-
.../commandhandler/CommandDispatcher.java | 4 +-
.../commandhandler/CommandHandler.java | 4 +-
.../CreatePipelineCommandHandler.java | 2 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 9 +-
.../DeleteContainerCommandHandler.java | 5 +-
.../FinalizeNewLayoutVersionCommandHandler.java | 4 +-
.../ReconstructECContainersCommandHandler.java | 2 +-
.../RefreshVolumeUsageCommandHandler.java | 2 +-
.../ReplicateContainerCommandHandler.java | 2 +-
.../SetNodeOperationalStateCommandHandler.java | 2 +-
.../states/datanode/RunningDatanodeState.java | 22 +-
.../states/endpoint/HeartbeatEndpointTask.java | 20 +-
.../states/endpoint/RegisterEndpointTask.java | 26 +-
.../states/endpoint/VersionEndpointTask.java | 11 +-
.../common/transport/server/XceiverServerGrpc.java | 1 -
.../common/transport/server/XceiverServerSpi.java | 1 -
.../common/transport/server/ratis/CSMMetrics.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 267 +-
.../transport/server/ratis/DispatcherContext.java | 30 +-
.../transport/server/ratis/XceiverServerRatis.java | 58 +-
.../container/common/utils/ContainerCache.java | 1 +
.../common/utils/ContainerCacheMetrics.java | 9 +-
.../container/common/utils/DatanodeStoreCache.java | 1 -
.../container/common/utils/DiskCheckUtil.java | 27 +-
.../container/common/utils/HddsVolumeUtil.java | 5 +-
.../common/volume/AvailableSpaceFilter.java | 39 +-
.../volume/CapacityVolumeChoosingPolicy.java | 2 +-
.../ozone/container/common/volume/DbVolume.java | 7 +-
.../ozone/container/common/volume/HddsVolume.java | 82 +-
.../container/common/volume/MutableVolumeSet.java | 76 +-
.../volume/RoundRobinVolumeChoosingPolicy.java | 4 +-
.../container/common/volume/StorageVolume.java | 187 +-
.../common/volume/StorageVolumeChecker.java | 2 +-
.../common/volume/ThrottledAsyncChecker.java | 9 +-
.../common/volume/VolumeChoosingPolicyFactory.java | 11 +-
.../ozone/container/common/volume/VolumeInfo.java | 191 -
.../container/common/volume/VolumeInfoMetrics.java | 6 +-
.../ozone/container/common/volume/VolumeUsage.java | 128 +-
.../ECReconstructionCoordinator.java | 4 +-
.../container/keyvalue/KeyValueContainer.java | 168 +-
.../container/keyvalue/KeyValueContainerCheck.java | 3 +-
.../KeyValueContainerMetadataInspector.java | 15 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 25 +-
.../container/keyvalue/TarContainerPacker.java | 111 +-
.../container/keyvalue/helpers/BlockUtils.java | 1 +
.../container/keyvalue/helpers/ChunkUtils.java | 1 -
.../helpers/KeyValueContainerLocationUtil.java | 1 +
.../keyvalue/helpers/KeyValueContainerUtil.java | 6 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 2 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 8 +-
.../keyvalue/impl/ChunkManagerFactory.java | 2 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 6 +-
.../keyvalue/impl/FilePerChunkStrategy.java | 6 +-
.../keyvalue/impl/KeyValueStreamDataChannel.java | 13 +-
.../keyvalue/impl/MappedBufferManager.java | 2 -
.../statemachine/background/BlockDeletingTask.java | 1 -
.../container/metadata/AbstractDatanodeStore.java | 2 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 1 +
.../metadata/DatanodeSchemaTwoDBDefinition.java | 12 +-
.../metadata/DatanodeStoreSchemaThreeImpl.java | 70 +-
.../DatanodeStoreWithIncrementalChunkList.java | 1 -
.../metadata/WitnessedContainerDBDefinition.java | 8 +-
.../metadata/WitnessedContainerMetadataStore.java | 3 +-
.../WitnessedContainerMetadataStoreImpl.java | 7 +-
.../AbstractBackgroundContainerScanner.java | 23 +-
.../ozoneimpl/BackgroundContainerDataScanner.java | 2 +-
.../BackgroundContainerMetadataScanner.java | 2 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 13 +-
.../ozoneimpl/OnDemandContainerDataScanner.java | 2 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 54 +-
.../replication/AbstractReplicationTask.java | 27 +-
.../container/replication/ContainerImporter.java | 24 +-
.../replication/CopyContainerCompression.java | 10 +-
.../replication/DownloadAndImportReplicator.java | 25 +-
.../replication/GrpcContainerUploader.java | 1 +
.../replication/GrpcReplicationClient.java | 3 +-
.../replication/SendContainerRequestHandler.java | 69 +-
.../replication/SimpleContainerDownloader.java | 2 +-
.../container/stream/DirstreamClientHandler.java | 15 +-
.../container/stream/DirstreamServerHandler.java | 3 +-
.../ozone/container/stream/StreamingClient.java | 1 -
.../upgrade/DataNodeUpgradeFinalizer.java | 4 +-
.../upgrade/DatanodeSchemaV2FinalizeAction.java | 2 +-
.../upgrade/DatanodeSchemaV3FinalizeAction.java | 2 +-
.../protocol/StorageContainerNodeProtocol.java | 6 +-
.../protocol/commands/CommandForDatanode.java | 5 +
.../ozone/protocol/commands/CommandStatus.java | 1 +
.../protocol/commands/DeleteBlocksCommand.java | 1 -
.../commands/ReconcileContainerCommand.java | 1 -
.../commands/ReconstructECContainersCommand.java | 1 +
.../hadoop/ozone/protocol/commands/SCMCommand.java | 1 +
.../ozone/protocolPB/ReconDatanodeProtocolPB.java | 4 +-
.../StorageContainerDatanodeProtocolPB.java | 4 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 3 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 10 +-
.../checksum/ContainerMerkleTreeTestUtils.java | 9 +-
.../ozone/container/common/ContainerTestUtils.java | 33 +-
.../ozone/container/common/SCMTestUtils.java | 14 +-
.../hadoop/ozone/container/common/ScmTestMock.java | 17 +-
.../container/common/TestBlockDeletingService.java | 41 +-
.../ozone/container/common/TestContainerCache.java | 13 +-
.../common/TestContainerLayoutVersion.java | 1 -
.../container/common/TestDatanodeStateMachine.java | 16 +-
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../TestSchemaTwoBackwardsCompatibility.java | 9 +-
...stStaleRecoveringContainerScrubbingService.java | 4 +-
.../container/common/helpers/TestBlockData.java | 4 +-
.../common/helpers/TestContainerUtils.java | 92 +-
.../common/helpers/TestDatanodeIdYaml.java | 2 +
.../common/impl/ContainerImplTestUtils.java | 32 +-
.../common/impl/TestContainerDataYaml.java | 11 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 17 +-
.../common/impl/TestContainerPersistence.java | 175 +-
.../container/common/impl/TestContainerSet.java | 9 +-
.../container/common/impl/TestHddsDispatcher.java | 88 +-
.../container/common/interfaces/TestHandler.java | 21 +-
.../statemachine/TestDatanodeConfiguration.java | 65 +
.../common/statemachine/TestStateContext.java | 17 +-
.../TestCloseContainerCommandHandler.java | 3 +-
.../TestDeleteBlocksCommandHandler.java | 5 +-
.../TestDeleteContainerCommandHandler.java | 1 +
.../TestReconcileContainerCommandHandler.java | 3 +-
.../server/ratis/TestContainerStateMachine.java | 309 +
.../ratis/TestContainerStateMachineFollower.java} | 11 +-
.../ratis/TestContainerStateMachineLeader.java} | 11 +-
.../common/volume/TestPeriodicVolumeChecker.java | 2 +-
.../common/volume/TestReservedVolumeSpace.java | 42 +-
.../common/volume/TestStorageVolumeChecker.java | 2 +-
.../volume/TestStorageVolumeHealthChecks.java | 46 +
.../container/common/volume/TestVolumeSet.java | 15 +-
.../common/volume/TestVolumeSetDiskChecks.java | 20 +-
.../keyvalue/ContainerLayoutTestInfo.java | 4 +-
.../keyvalue/ContainerTestVersionInfo.java | 40 +-
.../keyvalue/TestKeyValueBlockIterator.java | 1 -
.../container/keyvalue/TestKeyValueContainer.java | 50 +-
.../TestKeyValueContainerIntegrityChecks.java | 3 +-
.../TestKeyValueContainerMarkUnhealthy.java | 2 +-
.../TestKeyValueContainerMetadataInspector.java | 22 +-
.../container/keyvalue/TestKeyValueHandler.java | 46 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 5 -
.../container/keyvalue/TestTarContainerPacker.java | 62 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 2 +-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 8 +-
.../impl/TestKeyValueStreamDataChannel.java | 3 +-
.../TestBackgroundContainerDataScanner.java | 9 +-
.../TestBackgroundContainerMetadataScanner.java | 5 +-
.../container/ozoneimpl/TestContainerReader.java | 33 +-
.../ozoneimpl/TestContainerScannersAbstract.java | 1 -
.../TestOnDemandContainerDataScanner.java | 1 +
.../ReplicationSupervisorScheduling.java | 5 +-
.../replication/TestContainerImporter.java | 27 +-
.../TestCopyContainerResponseStream.java | 1 +
.../replication/TestGrpcContainerUploader.java | 2 +-
.../replication/TestGrpcReplicationService.java | 3 +-
.../container/replication/TestPushReplicator.java | 1 -
.../replication/TestReplicationSupervisor.java | 153 +-
.../replication/TestSendContainerOutputStream.java | 1 +
.../TestSendContainerRequestHandler.java | 17 +-
.../stream/TestDirstreamClientHandler.java | 3 -
.../container/stream/TestStreamingServer.java | 1 +
.../ozone/container/upgrade/UpgradeTestHelper.java | 4 +-
hadoop-hdds/crypto-api/pom.xml | 4 +-
hadoop-hdds/crypto-default/pom.xml | 4 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 8 +
hadoop-hdds/docs/content/feature/ErasureCoding.md | 2 +-
hadoop-hdds/docs/content/security/SecuringS3.md | 45 +
hadoop-hdds/docs/content/start/FromSource.md | 91 +-
hadoop-hdds/docs/content/start/Kubernetes.md | 32 +-
hadoop-hdds/docs/content/tools/AuditParser.md | 8 +-
hadoop-hdds/docs/content/tools/AuditParser.zh.md | 8 +-
hadoop-hdds/docs/pom.xml | 8 +-
hadoop-hdds/erasurecode/pom.xml | 4 +-
.../rawcoder/AbstractNativeRawDecoder.java | 2 +-
.../rawcoder/AbstractNativeRawEncoder.java | 2 +-
.../ozone/erasurecode/rawcoder/CoderUtil.java | 3 -
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 3 +-
.../rawcoder/XORRawErasureCoderFactory.java | 1 -
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 2 +-
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 4 +-
.../erasurecode/rawcoder/util/GaloisField.java | 17 +-
.../org/apache/ozone/erasurecode/DumpUtil.java | 4 +-
.../ozone/erasurecode/TestCodecRegistry.java | 4 +
.../apache/ozone/erasurecode/TestCoderBase.java | 37 +-
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 1 +
.../erasurecode/rawcoder/TestXORRawCoder.java | 2 -
.../framework/dev-support/findbugsExcludeFile.xml | 10 -
hadoop-hdds/framework/pom.xml | 9 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 144 +-
.../hadoop/hdds/fs/AbstractSpaceUsageSource.java | 0
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 0
.../main/java/org/apache/hadoop/hdds/fs/DU.java | 0
.../java/org/apache/hadoop/hdds/fs/DUFactory.java | 0
.../hadoop/hdds/fs/DedicatedDiskSpaceUsage.java | 2 +-
.../hdds/fs/DedicatedDiskSpaceUsageFactory.java | 0
.../hadoop/hdds/fs/SaveSpaceUsageToFile.java | 3 +-
.../hadoop/hdds/fs/SpaceUsageCheckFactory.java | 4 +-
.../hadoop/hdds/fs/SpaceUsageCheckParams.java | 0
.../hadoop/hdds/fs/SpaceUsagePersistence.java | 0
.../org/apache/hadoop/hdds/fs}/package-info.java | 4 +-
.../hdds/protocol/SecretKeyProtocolDatanode.java | 4 +-
.../ReconfigureProtocolClientSideTranslatorPB.java | 2 +-
.../protocolPB/ReconfigureProtocolDatanodePB.java | 4 +-
.../SCMSecurityProtocolClientSideTranslatorPB.java | 1 +
.../protocolPB/SecretKeyProtocolDatanodePB.java | 4 +-
.../DeletedBlocksTransactionInfoWrapper.java | 1 -
.../container/common/helpers/MoveDataNodePair.java | 8 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 67 +-
.../apache/hadoop/hdds/scm/ha}/package-info.java | 4 +-
.../apache/hadoop/hdds/scm/metadata/Replicate.java | 1 +
...lockLocationProtocolClientSideTranslatorPB.java | 3 +-
...inerLocationProtocolClientSideTranslatorPB.java | 27 +-
.../SCMBlockLocationFailoverProxyProvider.java | 267 +-
.../SCMContainerLocationFailoverProxyProvider.java | 271 +-
...ider.java => SCMFailoverProxyProviderBase.java} | 158 +-
.../SCMSecurityProtocolFailoverProxyProvider.java | 269 +-
.../SecretKeyProtocolFailoverProxyProvider.java | 263 +-
.../SingleSecretKeyProtocolProxyProvider.java | 11 +
.../hadoop/hdds/security/OzoneSecretManager.java | 2 +-
.../security/symmetric/DefaultSecretKeyClient.java | 2 -
.../security/token/ContainerTokenGenerator.java | 26 +-
.../hdds/security/x509/certificate/CertInfo.java | 8 +-
.../certificate/authority/CertificateApprover.java | 1 -
.../client/DefaultCertificateClient.java | 9 +-
.../certificate/client/SCMCertificateClient.java | 1 -
.../org/apache/hadoop/hdds/server/ServerUtils.java | 7 -
.../hadoop/hdds/server/ServiceRuntimeInfo.java | 2 +-
.../hadoop/hdds/server/events/EventQueue.java | 1 -
.../hadoop/hdds/server/events/EventWatcher.java | 6 +-
.../hadoop/hdds/server/events/IEventInfo.java | 1 +
.../hadoop/hdds/server/http/BaseHttpServer.java | 12 +-
.../hadoop/hdds/server/http/HttpRequestLog.java | 106 -
.../hdds/server/http/HttpRequestLogAppender.java | 62 -
.../hadoop/hdds/server/http/HttpServer2.java | 80 +-
.../hdds/server/http/HttpServer2Metrics.java | 42 +-
.../hadoop/hdds/server/http/ProfileServlet.java | 12 +-
.../org/apache/hadoop/hdds/utils/Archiver.java | 160 +
.../hadoop/hdds/utils/DBCheckpointServlet.java | 30 +-
.../hadoop/hdds/utils/DecayRpcSchedulerUtil.java | 1 -
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 5 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 106 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 1 +
.../org/apache/hadoop/hdds/utils/NettyMetrics.java | 36 +-
.../hadoop/hdds/utils/ProtocolMessageMetrics.java | 5 +-
.../hadoop/hdds/utils/TableCacheMetrics.java | 40 +-
.../apache/hadoop/hdds/utils/TransactionInfo.java | 30 +-
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 7 +-
.../hdds/utils/db/DBColumnFamilyDefinition.java | 36 +-
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 12 +-
.../apache/hadoop/hdds/utils/db/DBDefinition.java | 1 -
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 5 +
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 13 +
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 26 +-
.../hadoop/hdds/utils/db/DumpFileLoader.java | 39 -
.../hadoop/hdds/utils/db/RDBBatchOperation.java | 20 +-
.../apache/hadoop/hdds/utils/db/RDBMetrics.java | 26 +-
.../hadoop/hdds/utils/db/RDBSstFileLoader.java | 22 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 26 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 61 +-
.../hdds/utils/db/RDBStoreCodecBufferIterator.java | 114 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 22 +-
.../apache/hadoop/hdds/utils/db/RawKeyValue.java | 7 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 194 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 4 -
.../apache/hadoop/hdds/utils/db/TableConfig.java | 9 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 66 +-
.../hadoop/hdds/utils/db/cache/CacheValue.java | 20 +-
.../hadoop/hdds/utils/db/cache/FullTableCache.java | 11 +-
.../hdds/utils/db/cache/PartialTableCache.java | 14 +-
.../hadoop/hdds/utils/db/cache/TableCache.java | 4 +-
.../hadoop/hdds/utils/db/cache/TableNoCache.java | 101 +
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 2 +
.../org/apache/hadoop/ozone/common/BlockGroup.java | 1 +
.../ozone/common/DeleteBlockGroupResult.java | 1 +
.../org/apache/hadoop/ozone/common/Storage.java | 2 -
.../apache/hadoop/ozone/common/StorageInfo.java | 34 +-
.../hadoop/ozone/grpc/metrics/GrpcMetrics.java | 58 +-
.../GrpcMetricsServerRequestInterceptor.java | 0
.../GrpcMetricsServerResponseInterceptor.java | 0
.../metrics/GrpcMetricsServerTransportFilter.java | 0
.../hadoop/ozone/grpc/metrics/package-info.java | 0
.../java/org/apache/hadoop/ozone/lease/Lease.java | 1 -
.../ozone/lease/LeaseAlreadyExistException.java | 0
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 0
.../apache/hadoop/ozone/lease/LeaseException.java | 0
.../hadoop/ozone/lease/LeaseExpiredException.java | 0
.../apache/hadoop/ozone/lease/LeaseManager.java | 0
.../lease/LeaseManagerNotRunningException.java | 0
.../hadoop/ozone/lease/LeaseNotFoundException.java | 0
.../apache/hadoop/ozone/lease/package-info.java | 0
.../upgrade/AbstractLayoutVersionManager.java | 6 +-
.../ozone/upgrade/BasicUpgradeFinalizer.java | 23 +-
.../DefaultUpgradeFinalizationExecutor.java | 2 +-
.../hadoop/ozone/upgrade/UpgradeFinalizer.java | 88 +-
.../hadoop/hdds/fs/MockSpaceUsageCheckFactory.java | 0
.../hadoop/hdds/fs/MockSpaceUsageCheckParams.java | 0
.../hadoop/hdds/fs/MockSpaceUsagePersistence.java | 0
.../hadoop/hdds/fs/MockSpaceUsageSource.java | 0
.../hdds/fs/TestCachingSpaceUsageSource.java | 2 +-
.../java/org/apache/hadoop/hdds/fs/TestDU.java | 0
.../org/apache/hadoop/hdds/fs/TestDUFactory.java | 0
.../hdds/fs/TestDedicatedDiskSpaceUsage.java | 0
.../fs/TestDedicatedDiskSpaceUsageFactory.java | 0
.../hadoop/hdds/fs/TestSaveSpaceUsageToFile.java | 0
.../hadoop/hdds/fs/TestSpaceUsageFactory.java | 4 +-
.../apache/hadoop/hdds/scm/ha}/TestSCMHAUtils.java | 3 +-
.../security/ssl/TestReloadingX509KeyManager.java | 2 +-
.../ssl/TestReloadingX509TrustManager.java | 2 +-
.../security/ssl/TestSSLConnectionWithReload.java | 1 +
.../security/symmetric/TestManagedSecretKey.java | 6 +-
.../security/symmetric/TestSecretKeyManager.java | 2 +-
.../token/TestOzoneBlockTokenIdentifier.java | 2 +-
.../hdds/security/token/TokenVerifierTests.java | 5 -
.../certificate/authority/TestDefaultCAServer.java | 44 +-
.../certificate/authority/TestDefaultProfile.java | 2 -
.../client/TestDefaultCertificateClient.java | 11 +-
.../client/TestDnCertificateClientInit.java | 1 -
.../client/TestRootCaRotationPoller.java | 6 +-
.../certificate/utils/TestCertificateCodec.java | 6 +-
.../utils/TestCertificateSignRequest.java | 20 +-
.../hdds/server/events/TestEventQueueChain.java | 1 -
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 4 +-
.../hdds/server/http/TestHttpRequestLog.java | 51 -
.../hdds/server/http/TestHttpServer2Metrics.java | 8 +-
.../server/http/TestRatisDropwizardExports.java | 2 +-
.../hdds/server/http/TestRatisNameRewrite.java | 1 -
.../hadoop/hdds/utils/MapBackedTableIterator.java | 83 +
.../hadoop/hdds/utils/TestRDBSnapshotProvider.java | 7 +-
.../hadoop/hdds/utils/db/InMemoryTestTable.java | 3 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 12 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 51 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 81 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 60 +-
.../hadoop/hdds/utils/db/TestTypedTable.java | 147 +
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 46 +-
.../hadoop/ozone/audit/AuditLogTestUtils.java | 6 +-
.../org/apache/hadoop/ozone/audit/DummyEntity.java | 1 +
.../hadoop/ozone/lease/TestLeaseManager.java | 0
.../apache/hadoop/ozone/lease/package-info.java | 0
.../InjectedUpgradeFinalizationExecutor.java | 2 +-
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 19 +-
.../upgrade/TestLayoutVersionInstanceFactory.java | 4 -
.../ozone/upgrade/TestUpgradeFinalizerActions.java | 2 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 8 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
.../interface-admin/src/main/resources/proto.lock | 340 ++
hadoop-hdds/interface-client/pom.xml | 4 +-
.../src/main/proto/DatanodeClientProtocol.proto | 64 +-
.../interface-client/src/main/proto/hdds.proto | 12 +-
.../interface-client/src/main/resources/proto.lock | 272 +-
hadoop-hdds/interface-server/pom.xml | 4 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 10 -
.../interface-server/src/main/resources/proto.lock | 55 +-
hadoop-hdds/managed-rocksdb/pom.xml | 4 +-
.../hdds/utils/db/RocksDatabaseException.java} | 37 +-
.../db/managed/ManagedBlockBasedTableConfig.java | 1 +
.../hdds/utils/db/managed/ManagedRocksDB.java | 22 +-
.../utils/db/managed/ManagedRocksObjectUtils.java | 12 +-
.../apache/hadoop/hdds/utils/db}/package-info.java | 6 +-
hadoop-hdds/pom.xml | 5 +-
.../dev-support/findbugsExcludeFile.xml | 4 -
hadoop-hdds/rocks-native/pom.xml | 111 +-
hadoop-hdds/rocks-native/src/CMakeLists.txt | 2 +-
.../apache/hadoop/hdds/utils/NativeConstants.java | 2 +
.../db/managed/ManagedRawSSTFileIterator.java | 5 +
.../utils/db/managed/ManagedRawSSTFileReader.java | 10 +-
.../hadoop/hdds/utils/TestNativeLibraryLoader.java | 5 +-
.../org/apache/hadoop/hdds/utils/TestUtils.java | 2 +-
.../db/managed/TestManagedRawSSTFileIterator.java | 9 +-
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 9 +-
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 7 +-
.../compaction/log/TestCompactionLogEntry.java | 1 -
.../ozone/rocksdb/util/TestSstFileSetReader.java | 11 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 32 +-
.../server-scm/dev-support/findbugsExcludeFile.xml | 38 -
hadoop-hdds/server-scm/pom.xml | 5 +-
.../org/apache/hadoop/hdds/scm/FetchMetrics.java | 4 +-
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 1 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 19 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 8 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 4 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 21 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 2 +-
.../SCMDeletedBlockTransactionStatusManager.java | 9 +-
.../scm/block/ScmBlockDeletingServiceMetrics.java | 34 +-
.../container/AbstractContainerReportHandler.java | 324 +-
.../scm/container/CloseContainerEventHandler.java | 2 +-
.../hdds/scm/container/ContainerManager.java | 7 +-
.../hdds/scm/container/ContainerManagerImpl.java | 61 +-
.../hdds/scm/container/ContainerReplica.java | 4 +
.../hdds/scm/container/ContainerReportHandler.java | 87 +-
.../hdds/scm/container/ContainerStateManager.java | 34 +-
.../scm/container/ContainerStateManagerImpl.java | 70 +-
.../IncrementalContainerReportHandler.java | 77 +-
.../balancer/AbstractFindTargetGreedy.java | 12 +-
.../scm/container/balancer/ContainerBalancer.java | 3 +-
.../balancer/ContainerBalancerMetrics.java | 1 +
.../ContainerBalancerSelectionCriteria.java | 1 -
.../container/balancer/ContainerBalancerTask.java | 84 +-
.../hdds/scm/container/balancer/DataMoveInfo.java | 1 -
.../scm/container/balancer/FindSourceGreedy.java | 7 +-
.../FindTargetGreedyByNetworkTopology.java | 2 +-
.../balancer/FindTargetGreedyByUsageInfo.java | 2 +-
.../hdds/scm/container/balancer/MoveManager.java | 4 +-
.../metrics/SCMContainerManagerMetrics.java | 1 -
.../ContainerPlacementPolicyFactory.java | 1 -
.../ContainerPlacementStatusDefault.java | 1 -
.../algorithms/SCMContainerPlacementCapacity.java | 2 +-
.../algorithms/SCMContainerPlacementRackAware.java | 6 +-
.../SCMContainerPlacementRackScatter.java | 6 +-
.../algorithms/SCMContainerPlacementRandom.java | 2 +-
.../placement/metrics/SCMPerformanceMetrics.java | 14 +
.../replication/ContainerHealthResult.java | 7 +-
.../replication/ContainerReplicaPendingOps.java | 5 +-
.../replication/ECContainerReplicaCount.java | 15 +-
.../replication/ECOverReplicationHandler.java | 6 +-
.../replication/ECUnderReplicationHandler.java | 4 +-
.../replication/MisReplicationHandler.java | 2 +-
.../QuasiClosedStuckOverReplicationHandler.java | 124 +
.../replication/QuasiClosedStuckReplicaCount.java | 187 +
.../QuasiClosedStuckUnderReplicationHandler.java | 158 +
.../replication/RatisContainerReplicaCount.java | 22 +-
.../replication/RatisOverReplicationHandler.java | 4 +-
.../replication/RatisUnderReplicationHandler.java | 2 +-
.../container/replication/ReplicationManager.java | 93 +-
.../ReplicationManagerEventHandler.java | 34 +-
.../replication/ReplicationManagerMetrics.java | 2 -
.../replication/ReplicationManagerUtil.java | 10 +-
.../container/replication/ReplicationQueue.java | 4 +
.../health/ClosedWithUnhealthyReplicasHandler.java | 2 +-
.../health/DeletingContainerHandler.java | 8 +-
.../health/ECReplicationCheckHandler.java | 2 +-
.../replication/health/EmptyContainerHandler.java | 4 +-
.../health/MismatchedReplicasHandler.java | 46 +-
.../health/QuasiClosedContainerHandler.java | 15 +-
.../health/QuasiClosedStuckReplicationCheck.java | 132 +
.../health/RatisReplicationCheckHandler.java | 8 +-
.../RatisUnhealthyReplicationCheckHandler.java | 2 +-
.../health/VulnerableUnhealthyReplicasHandler.java | 2 +-
.../container/report/ContainerReportValidator.java | 6 +-
.../scm/container/states/ContainerAttribute.java | 210 +-
.../hdds/scm/container/states/ContainerEntry.java | 55 +
.../scm/container/states/ContainerQueryKey.java | 98 -
.../scm/container/states/ContainerStateMap.java | 449 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 4 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 7 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 1 -
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 5 -
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 7 +-
.../scm/ha/SCMHATransactionBufferMonitorTask.java | 2 +-
.../apache/hadoop/hdds/scm/ha/SCMNodeDetails.java | 5 -
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 1 +
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 8 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 2 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 1 -
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 3 -
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 9 +-
.../scm/ha/StatefulServiceStateManagerImpl.java | 12 +-
.../apache/hadoop/hdds/scm/node/CommandQueue.java | 40 +-
.../hadoop/hdds/scm/node/DatanodeAdminMonitor.java | 4 +
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 6 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 21 +-
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 4 +-
.../hadoop/hdds/scm/node/NewNodeHandler.java | 4 -
.../hdds/scm/node/NodeDecommissionManager.java | 29 +-
.../hdds/scm/node/NodeDecommissionMetrics.java | 1 -
.../apache/hadoop/hdds/scm/node/NodeManager.java | 30 +-
.../hadoop/hdds/scm/node/NodeReportHandler.java | 4 -
.../hadoop/hdds/scm/node/NodeStateManager.java | 97 +-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 287 +-
.../node/ReadOnlyHealthyToHealthyNodeHandler.java | 7 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 177 +-
.../hdds/scm/node/SCMNodeStorageStatMap.java | 26 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 6 +-
.../hadoop/hdds/scm/node/states/DatanodeEntry.java | 61 +
.../node/states/NodeAlreadyExistsException.java | 13 +-
.../scm/node/states/NodeNotFoundException.java | 14 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 277 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 1 -
.../hdds/scm/pipeline/ECPipelineProvider.java | 13 +-
.../hdds/scm/pipeline/PipelineActionHandler.java | 10 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 1 -
.../hdds/scm/pipeline/PipelineManagerImpl.java | 12 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 8 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 10 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 14 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 57 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 5 -
.../hdds/scm/pipeline/SCMPipelineMetrics.java | 8 +
.../scm/pipeline/WritableContainerFactory.java | 2 +-
.../scm/pipeline/WritableECContainerProvider.java | 6 +-
.../pipeline/WritableRatisContainerProvider.java | 8 +-
.../algorithms/CapacityPipelineChoosePolicy.java | 1 +
.../algorithms/HealthyPipelineChoosePolicy.java | 2 +-
.../algorithms/RoundRobinPipelineChoosePolicy.java | 2 +-
.../algorithms/LeaderChoosePolicyFactory.java | 1 -
.../SCMSecurityProtocolServerSideTranslatorPB.java | 5 +-
...lockLocationProtocolServerSideTranslatorPB.java | 5 +-
.../SecretKeyProtocolServerSideTranslatorPB.java | 5 +-
...inerLocationProtocolServerSideTranslatorPB.java | 10 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 359 --
.../hdds/scm/safemode/DataNodeSafeModeRule.java | 11 +-
.../hdds/scm/safemode/ECContainerSafeModeRule.java | 249 +
.../scm/safemode/HealthyPipelineSafeModeRule.java | 4 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 1 -
.../scm/safemode/RatisContainerSafeModeRule.java | 200 +
.../hdds/scm/safemode/SCMSafeModeManager.java | 32 +-
.../hadoop/hdds/scm/safemode/SafeModeMetrics.java | 5 +-
.../hdds/scm/safemode/SafeModeRuleFactory.java | 31 +-
.../scm/security/RootCARotationHandlerImpl.java | 13 +-
.../hdds/scm/security/RootCARotationManager.java | 3 +-
.../scm/security/ScmSecretKeyStateBuilder.java | 12 +-
.../hdds/scm/security/SecretKeyManagerService.java | 3 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 15 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 17 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 592 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 13 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 17 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 2 +-
.../hdds/scm/server/SCMStarterInterface.java | 3 +
.../hdds/scm/server/StorageContainerManager.java | 31 +-
.../scm/server/upgrade/FinalizationCheckpoint.java | 16 +-
.../scm/server/upgrade/FinalizationManager.java | 6 +-
.../server/upgrade/FinalizationManagerImpl.java | 8 +-
.../upgrade/FinalizationStateManagerImpl.java | 17 +-
.../ScmOnFinalizeActionForDatanodeSchemaV2.java | 2 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 14 +
.../commands/RetriableDatanodeEventWatcher.java | 2 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 52 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 2 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 60 +-
.../scm/TestStorageContainerManagerHttpServer.java | 2 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 4 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 10 +-
.../command/TestCommandStatusReportHandler.java | 3 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 96 +-
.../hdds/scm/container/Node2ContainerMap.java | 165 -
.../hdds/scm/container/SimpleMockNodeManager.java | 42 +-
.../scm/container/TestContainerManagerImpl.java | 43 +-
.../scm/container/TestContainerReportHandler.java | 411 +-
.../scm/container/TestContainerStateManager.java | 30 +-
.../TestIncrementalContainerReportHandler.java | 119 +-
.../hdds/scm/container/TestNode2ContainerMap.java | 308 -
.../scm/container/TestUnknownContainerReport.java | 2 +-
.../container/balancer/TestContainerBalancer.java | 6 +-
.../TestContainerBalancerDatanodeNodeLimit.java | 3 +-
.../balancer/TestContainerBalancerTask.java | 14 +-
.../scm/container/balancer/TestMoveManager.java | 6 +-
.../algorithms/TestContainerPlacementFactory.java | 18 +-
.../TestSCMContainerPlacementCapacity.java | 22 +-
.../TestSCMContainerPlacementRackAware.java | 44 +-
.../TestSCMContainerPlacementRackScatter.java | 43 +-
.../TestSCMContainerPlacementRandom.java | 24 +-
.../container/replication/ReplicationTestUtil.java | 67 +-
.../TestContainerReplicaPendingOps.java | 154 +-
.../replication/TestECMisReplicationHandler.java | 1 -
.../replication/TestECOverReplicationHandler.java | 6 +-
.../replication/TestECUnderReplicationHandler.java | 13 +-
.../replication/TestMisReplicationHandler.java | 5 +-
...TestQuasiClosedStuckOverReplicationHandler.java | 170 +
.../TestQuasiClosedStuckReplicaCount.java | 350 ++
...estQuasiClosedStuckUnderReplicationHandler.java | 210 +
.../TestRatisMisReplicationHandler.java | 1 -
.../TestRatisOverReplicationHandler.java | 9 +-
.../TestRatisUnderReplicationHandler.java | 2 +-
.../replication/TestReplicationManager.java | 188 +-
.../TestReplicationManagerEventHandler.java | 72 +
.../TestReplicationManagerScenarios.java | 57 +-
.../replication/TestReplicationManagerUtil.java | 8 +-
.../health/TestDeletingContainerHandler.java | 25 +-
.../health/TestMismatchedReplicasHandler.java | 61 +-
.../health/TestOpenContainerHandler.java | 1 +
.../TestQuasiClosedStuckReplicationCheck.java | 292 +
.../TestRatisUnhealthyReplicationCheckHandler.java | 2 -
.../TestVulnerableUnhealthyReplicasHandler.java | 8 +-
.../container/states/TestContainerAttribute.java | 109 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 12 +-
.../hadoop/hdds/scm/ha/TestSCMConfiguration.java | 1 -
.../hadoop/hdds/scm/ha/TestSCMHAMetrics.java | 4 +-
.../hadoop/hdds/scm/node/ScmNodeTestUtil.java} | 20 +-
.../hadoop/hdds/scm/node/TestCommandQueue.java | 2 +-
.../hdds/scm/node/TestContainerPlacement.java | 11 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 38 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 25 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 37 +-
.../hdds/scm/node/TestNodeDecommissionMetrics.java | 23 +-
.../hdds/scm/node/TestNodeReportHandler.java | 4 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 14 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 110 +-
.../hadoop/hdds/scm/node/TestSCMNodeMetrics.java | 6 +-
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 5 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 4 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 118 +-
.../TestCreateForReadComparator.java} | 38 +-
.../hdds/scm/pipeline/TestECPipelineProvider.java | 8 +-
.../scm/pipeline/TestPipelineActionHandler.java | 9 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 30 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 6 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 9 -
.../hdds/scm/pipeline/TestPipelineStateMap.java | 2 -
.../scm/pipeline/TestRatisPipelineProvider.java | 27 +
.../TestWritableRatisContainerProvider.java | 2 +-
.../TestPipelineChoosePolicyFactory.java | 2 -
.../scm/safemode/TestDataNodeSafeModeRule.java | 172 +
.../safemode/TestHealthyPipelineSafeModeRule.java | 20 +-
.../TestOneReplicaPipelineSafeModeRule.java | 15 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 177 +-
.../hdds/scm/safemode/TestSafeModeRuleFactory.java | 6 +-
.../scm/security/TestRootCARotationManager.java | 16 +-
.../scm/server/TestSCMBlockProtocolServer.java | 6 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 13 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 1 -
.../hdds/scm/upgrade/TestScmFinalization.java | 8 +-
.../ozone/container/common/TestEndPoint.java | 60 +-
.../testutils/ReplicationNodeManagerMock.java | 537 --
.../ozone/container/testutils/package-info.java | 19 -
.../replicationManagerTests/quasi_closed.json | 163 +-
hadoop-hdds/test-utils/pom.xml | 79 +-
.../java/org/apache/ozone/test/tag/Native.java | 37 -
.../org/apache/ozone/test/GenericTestUtils.java | 60 +-
.../org/apache/ozone/test}/InputSubstream.java | 7 +-
.../java/org/apache/ozone/test/IntLambda.java | 0
.../java/org/apache/ozone/test/JacocoServer.java | 5 +-
.../org/apache/ozone/test/LambdaTestUtils.java | 5 +-
.../java/org/apache/ozone/test/Log4j1Capturer.java | 0
.../java/org/apache/ozone/test/Log4j2Capturer.java | 0
.../java/org/apache/ozone/test/MetricsAsserts.java | 0
.../java/org/apache/ozone/test/OzoneTestBase.java | 0
.../java/org/apache/ozone/test/SpyInputStream.java | 0
.../org/apache/ozone/test/SpyOutputStream.java | 0
.../java/org/apache/ozone/test/TestClock.java | 0
.../apache/ozone/test/TimedOutTestsListener.java | 0
.../java/org/apache/ozone/test/package-info.java | 0
.../java/org/apache/ozone/test/tag/Flaky.java | 0
.../java/org/apache/ozone/test/tag/Slow.java | 0
.../java/org/apache/ozone/test/tag/Unhealthy.java | 0
.../org/apache/ozone/test/tag/package-info.java | 0
...g.junit.platform.launcher.TestExecutionListener | 0
hadoop-hdds/tools/pom.xml | 6 +-
.../org/apache/hadoop/hdds/cli/ItemsFromStdin.java | 67 +
.../scm/cli/ContainerBalancerStatusSubcommand.java | 11 +-
.../hdds/scm/cli/ContainerOperationClient.java | 11 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 21 +-
.../hdds/scm/cli/SafeModeWaitSubcommand.java | 5 +-
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 4 +-
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 2 +-
.../hadoop/hdds/scm/cli/cert/ListSubcommand.java | 2 +-
.../scm/cli/container/ContainerIDParameters.java | 25 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 47 +-
.../hdds/scm/cli/container/ListSubcommand.java | 129 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 2 +-
.../hdds/scm/cli/container/UpgradeSubcommand.java | 162 +-
.../scm/cli/container/upgrade/UpgradeManager.java | 172 -
.../scm/cli/container/upgrade/UpgradeTask.java | 476 --
.../scm/cli/container/upgrade/UpgradeUtils.java | 86 -
.../hdds/scm/cli/datanode/HostNameParameters.java | 27 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 1 -
.../scm/cli/datanode/MaintenanceSubCommand.java | 1 -
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 4 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 2 -
.../scm/cli/container/TestReportSubCommand.java | 2 +-
.../cli/container/upgrade/TestUpgradeManager.java | 304 -
.../datanode/TestContainerBalancerSubCommand.java | 226 +-
.../datanode/TestDecommissionStatusSubCommand.java | 13 +-
hadoop-ozone/cli-shell/pom.xml | 4 +-
.../org/apache/hadoop/ozone/shell/Handler.java | 3 -
.../apache/hadoop/ozone/shell/OzoneAddress.java | 38 +-
.../java/org/apache/hadoop/ozone/shell/Shell.java | 22 +-
.../hadoop/ozone/shell/bucket/BucketUri.java | 2 +-
.../ozone/shell/bucket/DeleteBucketHandler.java | 2 +-
.../hadoop/ozone/shell/common/VolumeBucketUri.java | 2 +-
.../hadoop/ozone/shell/keys/GetKeyHandler.java | 7 +-
.../org/apache/hadoop/ozone/shell/keys/KeyUri.java | 2 +-
.../hadoop/ozone/shell/prefix/PrefixUri.java | 2 +-
.../shell/snapshot/ListSnapshotDiffHandler.java | 28 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 2 -
.../hadoop/ozone/shell/snapshot/SnapshotUri.java | 2 +-
.../ozone/shell/tenant/GetUserInfoHandler.java | 2 +-
.../hadoop/ozone/shell/token/TokenOption.java | 9 +-
.../ozone/shell/volume/DeleteVolumeHandler.java | 2 +-
.../hadoop/ozone/shell/TestOzoneAddress.java | 2 +-
.../client/dev-support/findbugsExcludeFile.xml | 5 -
hadoop-ozone/client/pom.xml | 5 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 83 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 3 +-
.../apache/hadoop/ozone/client/OzoneClient.java | 1 +
.../hadoop/ozone/client/OzoneClientFactory.java | 5 -
.../hadoop/ozone/client/OzoneKeyLocation.java | 1 +
.../apache/hadoop/ozone/client/OzoneVolume.java | 1 +
.../org/apache/hadoop/ozone/client/VolumeArgs.java | 1 +
.../client/checksum/BaseFileChecksumHelper.java | 4 +-
.../client/checksum/ECBlockChecksumComputer.java | 5 +-
.../checksum/ReplicatedBlockChecksumComputer.java | 2 +-
.../client/io/BlockDataStreamOutputEntry.java | 1 -
.../client/io/BlockDataStreamOutputEntryPool.java | 10 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 3 +-
.../client/io/BlockOutputStreamEntryPool.java | 5 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 9 +-
.../ozone/client/io/KeyDataStreamOutput.java | 5 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 2 +-
.../ozone/client/io/KeyOutputStreamSemaphore.java | 2 +-
.../ozone/client/io/OzoneCryptoInputStream.java | 2 +-
.../hadoop/ozone/client/io/OzoneInputStream.java | 1 +
.../ozone/client/protocol/ClientProtocol.java | 19 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 23 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 2 +-
.../hadoop/ozone/client/MockOmTransport.java | 1 -
.../client/MultiNodePipelineBlockAllocator.java | 3 +-
.../TestBlockOutputStreamIncrementalPutBlock.java | 2 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 39 +-
.../TestReplicatedBlockChecksumComputer.java | 4 +-
.../ozone/client/io/TestKeyOutputStream.java | 2 +-
.../hadoop/ozone/client/rpc/TestRpcClient.java | 1 +
hadoop-ozone/common/pom.xml | 5 +-
.../apache/hadoop/hdds/protocol/StorageType.java | 2 +-
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 2 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 22 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 19 +-
.../apache/hadoop/ozone/conf/OMClientConfig.java | 1 -
.../apache/hadoop/ozone/om/IOmMetadataReader.java | 1 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 44 +-
.../java/org/apache/hadoop/ozone/om/OmConfig.java | 42 +-
.../apache/hadoop/ozone/om/OmUpgradeConfig.java | 2 -
.../hadoop/ozone/om/exceptions/OMException.java | 1 +
.../ozone/om/exceptions/OMNotLeaderException.java | 4 -
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 2 +-
.../om/ha/HadoopRpcOMFailoverProxyProvider.java | 3 +-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 7 +-
.../ozone/om/helpers/MultipartUploadKeys.java | 92 -
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 2 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 3 -
.../hadoop/ozone/om/helpers/OmDeleteKeys.java | 1 -
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 12 +-
.../ozone/om/helpers/OmMultipartAbortInfo.java | 1 -
.../ozone/om/helpers/OmMultipartUploadList.java | 56 +-
.../hadoop/ozone/om/helpers/OmRangerSyncArgs.java | 1 +
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 8 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 1 -
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 1 -
.../hadoop/ozone/om/helpers/ServiceInfo.java | 1 -
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 49 +-
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 1 +
.../hadoop/ozone/om/lock/IOzoneManagerLock.java | 12 +
.../apache/hadoop/ozone/om/lock/OMLockDetails.java | 1 -
.../hadoop/ozone/om/lock/OmReadOnlyLock.java | 21 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 148 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 1 -
.../hadoop/ozone/om/protocol/OMAdminProtocol.java | 6 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 47 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 5 +-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 5 -
.../protocolPB/OMAdminProtocolClientSideImpl.java | 22 +-
.../OMInterServiceProtocolClientSideImpl.java | 5 -
.../om/protocolPB/OzoneManagerClientProtocol.java | 2 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 51 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 2 -
.../ozone/security/acl/IAccessAuthorizer.java | 2 +
.../hadoop/ozone/security/acl/OzoneObjInfo.java | 1 -
.../hadoop/ozone/security/acl/RequestContext.java | 1 -
.../snapshot/ListSnapshotDiffJobResponse.java | 39 +-
.../ozone/snapshot/SnapshotDiffReportOzone.java | 5 +-
.../ozone/snapshot/SnapshotDiffResponse.java | 39 +
.../org/apache/hadoop/ozone/util/PayloadUtils.java | 1 +
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 4 +-
.../org/apache/hadoop/ozone/om/TestOmConfig.java | 40 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 2 +-
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 10 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 12 +-
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 8 +-
.../hadoop/ozone/om/helpers/TestOzoneAclUtil.java | 20 +-
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 33 +-
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 1 -
.../ozone/security/TestGDPRSymmetricKey.java | 9 +-
.../security/TestOzoneDelegationTokenSelector.java | 8 +-
.../ozone/security/acl/TestOzoneObjInfo.java | 1 -
.../apache/hadoop/ozone/util/TestRadixTree.java | 2 -
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/datanode/pom.xml | 4 +-
.../dev-support/checks/_mvn_unit_report.sh | 17 +-
hadoop-ozone/dev-support/checks/_post_process.sh | 8 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 2 +-
hadoop-ozone/dev-support/checks/bats.sh | 1 +
hadoop-ozone/dev-support/checks/coverage.sh | 2 +-
hadoop-ozone/dev-support/checks/junit.sh | 8 +-
hadoop-ozone/dev-support/checks/license.sh | 2 +-
hadoop-ozone/dev-support/checks/native.sh | 23 -
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dev-support/intellij/install-runconfigs.sh | 21 -
hadoop-ozone/dev-support/intellij/ozone-style.xml | 71 -
.../dist/dev-support/bin/dist-layout-stitching | 11 +-
hadoop-ozone/dist/pom.xml | 5 +-
.../dist/src/main/assemblies/ozone-src.xml | 8 +
.../dashboards/Ozone - DeleteKey Metrics.json | 6210 +++++++++++++++-----
.../grafana/dashboards/Ozone - OM Snapshot.json | 539 ++
.../dashboards/Ozone - Overall Metrics.json | 101 +-
...ics.json => Ozone - Ozone Manager RocksDB.json} | 2398 +++++---
.../dist/src/main/compose/common/hadoop-test.sh | 3 +-
.../dist/src/main/compose/common/s3-haproxy.yaml | 6 -
.../dist/src/main/compose/compatibility/test.sh | 2 +
.../compose/ozone-balancer/docker-compose.yaml | 14 +-
.../src/main/compose/ozone-balancer/test-ec.sh | 2 +
.../src/main/compose/ozone-balancer/test-ratis.sh | 2 +
.../dist/src/main/compose/ozone-csi/test.sh | 2 +
.../dist/src/main/compose/ozone-ha/test-hadoop.sh | 2 +
.../test-haproxy-s3g.sh} | 12 +-
.../dist/src/main/compose/ozone-ha/test.sh | 2 +
.../compose/ozone-om-prepare/docker-compose.yaml | 3 +
.../dist/src/main/compose/ozone-om-prepare/test.sh | 2 +
.../dist/src/main/compose/ozone-topology/test.sh | 2 +
.../dist/src/main/compose/ozone/test-ec.sh | 2 +
.../dist/src/main/compose/ozone/test-failures1.sh | 2 +
.../dist/src/main/compose/ozone/test-failures2.sh | 2 +
.../dist/src/main/compose/ozone/test-hadoop.sh | 2 +
.../src/main/compose/ozone/test-legacy-bucket.sh | 2 +
.../dist/src/main/compose/ozone/test-s3a.sh | 2 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 3 +
.../dist/src/main/compose/ozonescripts/test.sh | 2 +
.../compose/ozonesecure-ha/docker-compose.yaml | 125 +-
.../src/main/compose/ozonesecure-ha/s3-haproxy.cfg | 48 +
.../s3-haproxy.yaml} | 53 +-
.../s3g-port-forward.yml} | 13 +-
.../compose/ozonesecure-ha/s3g-virtual-host.yaml | 2 +-
.../compose/ozonesecure-ha/scm-decommission.yaml | 6 +-
.../test-haproxy-s3g.sh | 21 +-
.../main/compose/ozonesecure-ha/test-leadership.sh | 2 +
.../compose/ozonesecure-ha/test-om-bootstrap.sh | 2 +
.../src/main/compose/ozonesecure-ha/test-s3a.sh | 3 +
.../ozonesecure-ha/test-s3g-virtual-host.sh | 2 +
.../ozonesecure-ha/test-scm-decommission.sh | 2 +
.../dist/src/main/compose/ozonesecure-ha/test.sh | 4 +-
.../dist/src/main/compose/ozonesecure-mr/test.sh | 2 +
.../ozonesecure/test-certificate-rotation.sh | 2 +
.../dist/src/main/compose/ozonesecure/test-ec.sh | 2 +
.../dist/src/main/compose/ozonesecure/test-fcq.sh | 2 +
.../compose/ozonesecure/test-root-ca-rotation.sh | 2 +
.../src/main/compose/ozonesecure/test-vault.sh | 2 +
.../dist/src/main/compose/ozonesecure/test.sh | 4 +-
hadoop-ozone/dist/src/main/compose/restart/test.sh | 2 +
hadoop-ozone/dist/src/main/compose/test-all.sh | 3 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 45 +-
.../dist/src/main/compose/upgrade/README.md | 5 +-
.../main/compose/upgrade/compose/ha/docker-config | 2 +
.../test-hadoop.sh => upgrade/compose/non-ha/.env} | 16 +-
.../compose/non-ha}/docker-compose.yaml | 140 +-
.../upgrade/compose/{ha => non-ha}/docker-config | 48 +-
.../test.sh => upgrade/compose/non-ha/load.sh} | 19 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 8 +-
.../dist/src/main/compose/upgrade/testlib.sh | 2 +-
.../callbacks/{1.5.0 => 2.0.0}/callback.sh | 2 +-
hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 9 +-
.../dist/src/main/compose/xcompat/test-new.sh | 2 +
.../dist/src/main/compose/xcompat/test-old.sh | 2 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 17 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 66 +-
.../license/bin/licenses/LICENSE-org.jline.txt | 35 +
.../main/license/bin/licenses/NOTICE-hadoop.txt | 9 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 10 +-
.../src/main/smoketest/admincli/container.robot | 50 +-
.../main/smoketest/auditparser/auditparser.robot | 10 +-
.../src/main/smoketest/balancer/testBalancer.robot | 10 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 66 +-
.../dist/src/main/smoketest/compatibility/om.robot | 10 +-
.../src/main/smoketest/compatibility/recon.robot | 8 +-
.../debug/ozone-debug-corrupt-block.robot | 9 +-
.../debug/ozone-debug-dead-datanode.robot | 15 +-
.../debug/ozone-debug-stale-datanode.robot | 10 +-
.../smoketest/debug/ozone-debug-tests-ec3-2.robot | 45 +-
.../smoketest/debug/ozone-debug-tests-ec6-3.robot | 54 +-
.../main/smoketest/debug/ozone-debug-tests.robot | 11 +-
.../src/main/smoketest/debug/ozone-debug.robot | 10 +-
.../src/main/smoketest/freon/echoRPCLoad.robot | 2 +-
.../main/smoketest/hsync/upgrade-hsync-check.robot | 4 +-
.../src/main/smoketest/ozonefs/ozonefs-obs.robot | 84 +
.../src/main/smoketest/snapshot/snapshot-sh.robot | 15 +-
.../smoketest/upgrade/check-finalization.robot | 14 +-
.../dist/src/main/smoketest/upgrade/finalize.robot | 7 +-
.../dist/src/main/smoketest/upgrade/lib.robot | 66 +
.../dist/src/main/smoketest/upgrade/prepare.robot | 7 +-
hadoop-ozone/dist/src/shell/conf/log4j.properties | 19 +
hadoop-ozone/dist/src/shell/ozone/ozone | 23 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 15 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 48 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 2 +-
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 23 +-
.../ozone/TestDatanodeMiniChaosOzoneCluster.java | 12 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 79 +-
.../TestOzoneManagerMiniChaosOzoneCluster.java | 11 +-
...orageContainerManagerMiniChaosOzoneCluster.java | 11 +-
.../hadoop/ozone/failure/FailureManager.java | 5 +-
.../org/apache/hadoop/ozone/failure/Failures.java | 1 +
.../ozone/loadgenerators/AgedDirLoadGenerator.java | 2 +-
.../ozone/loadgenerators/AgedLoadGenerator.java | 4 +-
.../hadoop/ozone/loadgenerators/DataBuffer.java | 2 +-
.../loadgenerators/FilesystemLoadGenerator.java | 2 +-
.../hadoop/ozone/loadgenerators/LoadBucket.java | 4 +-
.../hadoop/ozone/loadgenerators/LoadExecutors.java | 3 +-
.../loadgenerators/NestedDirLoadGenerator.java | 2 +-
.../loadgenerators/RandomDirLoadGenerator.java | 2 +-
.../ozone/loadgenerators/RandomLoadGenerator.java | 6 +-
.../loadgenerators/ReadOnlyLoadGenerator.java | 3 +-
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
.../dev-support/findbugsExcludeFile.xml | 23 -
hadoop-ozone/httpfsgateway/pom.xml | 4 +-
.../apache/ozone/fs/http/server/FSOperations.java | 2 +
.../fs/http/server/HttpFSParametersProvider.java | 5 +-
.../apache/ozone/fs/http/server/HttpFSServer.java | 150 +-
.../ozone/fs/http/server/HttpFSServerWebApp.java | 24 +-
.../http/server/metrics/HttpFSServerMetrics.java | 5 +-
.../service/hadoop/FileSystemAccessService.java | 14 +-
.../instrumentation/InstrumentationService.java | 50 +-
.../org/apache/ozone/lib/servlet/ServerWebApp.java | 2 +-
.../main/java/org/apache/ozone/lib/util/Check.java | 2 +-
.../main/java/org/apache/ozone/lib/wsrs/Param.java | 3 +-
.../java/org/apache/ozone/lib/wsrs/Parameters.java | 2 +-
.../org/apache/ozone/lib/wsrs/StringParam.java | 2 +-
hadoop-ozone/httpfsgateway/src/site/site.xml | 2 +-
.../fs/http/server/metrics/TestHttpFSMetrics.java | 22 +-
hadoop-ozone/insight/pom.xml | 4 +-
.../org/apache/hadoop/ozone/insight/Component.java | 2 +-
.../ozone/insight/ConfigurationSubCommand.java | 6 +-
.../ozone/insight/datanode/RatisInsight.java | 2 +-
.../dev-support/findbugsExcludeFile.xml | 7 +-
.../pom.xml | 77 +-
.../apache/hadoop/ozone/s3/S3ClientFactory.java | 161 +
.../apache/hadoop/ozone/s3/S3GatewayService.java | 69 +
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 79 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 92 +-
.../hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java | 0
.../awssdk/v1/TestS3SDKV1WithRatisStreaming.java | 0
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 294 +
.../hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java} | 6 +-
.../awssdk/v2/TestS3SDKV2WithRatisStreaming.java} | 4 +-
.../src/test/resources/ozone-site.xml | 129 +
.../dev-support/findbugsExcludeFile.xml | 107 -
hadoop-ozone/integration-test/pom.xml | 41 +-
.../fs/contract/AbstractContractAppendTest.java | 4 -
.../fs/contract/AbstractContractConcatTest.java | 4 -
.../AbstractContractCopyFromLocalTest.java | 10 +-
.../fs/contract/AbstractContractCreateTest.java | 1 -
.../fs/contract/AbstractContractEtagTest.java | 3 +-
.../AbstractContractGetFileStatusTest.java | 1 -
.../AbstractContractMultipartUploaderTest.java | 3 +-
.../fs/contract/AbstractContractSeekTest.java | 4 +-
.../fs/contract/AbstractContractSetTimesTest.java | 4 -
.../fs/contract/AbstractFSContractTestBase.java | 1 -
.../hadoop/fs/contract/ContractTestUtils.java | 10 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 56 +-
.../ozone/AbstractOzoneFileSystemTestWithFSO.java | 6 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 99 +-
.../AbstractRootedOzoneFileSystemTestWithFSO.java | 1 +
.../hadoop/fs/ozone/OzoneFileSystemTests.java | 21 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 4 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 25 +-
.../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 22 +-
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 11 +-
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 23 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 95 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 139 +-
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 4 +-
.../fs/ozone/TestOzoneFileSystemMetrics.java | 86 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 47 +-
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 8 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 7 +-
.../apache/hadoop/hdds/scm/TestCloseContainer.java | 21 +-
.../apache/hadoop/hdds/scm/TestCommitInRatis.java | 6 +-
.../scm}/TestContainerOperations.java | 167 +-
.../hdds/scm/TestContainerReportWithKeys.java | 8 +-
.../hadoop/hdds/scm/TestFailoverWithSCMHA.java | 11 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 4 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 3 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 8 +-
.../hdds/scm/TestSCMInstallSnapshotWithHA.java | 19 +-
.../org/apache/hadoop/hdds/scm/TestSCMMXBean.java | 3 +-
.../hadoop/hdds/scm/TestSCMNodeManagerMXBean.java | 2 +-
.../hadoop/hdds/scm/TestSecretKeySnapshot.java | 8 +-
.../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 8 +-
.../hdds/scm/TestStorageContainerManager.java | 25 +-
.../hdds/scm/TestStorageContainerManagerHA.java | 236 +-
...estStorageContainerManagerHAWithAllRunning.java | 218 +
.../apache/hadoop/hdds/scm/TestWatchForCommit.java | 15 +-
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 6 +-
.../hadoop/hdds/scm/TestXceiverClientManager.java | 24 +-
.../TestContainerStateManagerIntegration.java | 76 +-
.../container/TestScmApplyTransactionFailure.java | 2 +-
.../metrics/TestSCMContainerManagerMetrics.java | 78 +-
.../TestReplicationManagerIntegration.java | 263 +
.../scm/node/TestDecommissionAndMaintenance.java | 31 +-
.../apache/hadoop/hdds/scm/node/TestQueryNode.java | 64 -
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 2 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 +
.../hdds/scm/pipeline/TestPipelineClose.java | 3 +-
.../TestRatisPipelineCreateAndDestroy.java | 2 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 5 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 5 -
.../hadoop/hdds/scm/storage/TestCommitWatcher.java | 3 -
.../hdds/scm/storage/TestContainerCommandsEC.java | 36 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 13 +-
.../hadoop/hdds/upgrade/TestHddsUpgradeUtils.java | 10 +-
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 5 +-
.../db/managed/TestRocksObjectLeakDetector.java | 2 +-
.../org/apache/hadoop/ozone/TestBlockTokens.java | 12 +-
.../apache/hadoop/ozone/TestBlockTokensCLI.java | 8 +-
.../org/apache/hadoop/ozone/TestCpuMetrics.java | 5 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 66 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 13 +-
.../ozone/TestGetClusterTreeInformation.java | 2 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 5 +-
.../hadoop/ozone/TestMultipartObjectGet.java | 4 +-
.../apache/hadoop/ozone/TestOMSortDatanodes.java | 5 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 5 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 35 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 560 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 13 +-
.../client/rpc/TestBlockDataStreamOutput.java | 2 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 12 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 16 +-
.../rpc/TestCloseContainerHandlingByClient.java | 84 +-
.../client/rpc/TestContainerStateMachine.java | 18 +-
.../rpc/TestContainerStateMachineFailures.java | 16 +-
.../rpc/TestContainerStateMachineStream.java | 206 +-
.../client/rpc/TestDeleteWithInAdequateDN.java | 2 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 2 +-
.../client/rpc/TestFailureHandlingByClient.java | 2 -
.../client/rpc/TestHybridPipelineOnDatanode.java | 38 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 55 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 80 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 10 +-
.../TestOzoneRpcClientWithKeyLatestVersion.java | 42 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 11 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 144 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 3 -
.../client/rpc/read/TestChunkInputStream.java | 10 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 7 +-
.../ozone/container/TestContainerReplication.java | 77 +-
.../container/TestContainerReportHandling.java | 51 +-
.../TestContainerReportHandlingWithHA.java | 68 +-
.../ozone/container/TestECContainerRecovery.java | 6 +-
.../apache/hadoop/ozone/container/TestHelper.java | 32 +-
.../commandhandler/TestBlockDeletion.java | 30 +-
.../commandhandler/TestDeleteContainerHandler.java | 28 +-
.../commandhandler/TestFinalizeBlock.java | 25 +-
.../transport/server/ratis/TestCSMMetrics.java | 4 +-
.../container/metrics/TestContainerMetrics.java | 30 +-
.../container/ozoneimpl/TestOzoneContainer.java | 6 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 16 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 7 +-
.../container/server/TestContainerServer.java | 27 +-
.../server/TestSecureContainerServer.java | 30 +-
.../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 11 +-
.../TestContainerCommandReconciliation.java | 8 +-
.../TestContainerScannerIntegrationAbstract.java | 3 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 81 +-
.../TestDatanodeHddsVolumeFailureToleration.java | 7 +-
.../hadoop/ozone/freon/OmBucketTestUtils.java | 215 +
.../ozone/freon/TestHadoopDirTreeGenerator.java | 152 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 123 +-
.../hadoop/ozone/freon/TestHsyncGenerator.java | 57 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 372 +-
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 337 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 121 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 1 -
.../apache/hadoop/ozone/om/OMUpgradeTestUtils.java | 6 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 28 +-
.../org/apache/hadoop/ozone/om/TestBucket.java | 13 +-
.../ozone/om/TestBucketLayoutWithOlderClient.java | 3 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 25 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 59 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 13 +-
.../org/apache/hadoop/ozone/om/TestListKeys.java | 35 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 52 +-
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 6 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 22 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 66 +-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 4 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 68 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 6 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 54 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 25 +-
.../ozone/om/TestOmContainerLocationCache.java | 1 -
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 58 +-
.../ozone/om/TestOmStartupSlvLessThanMlv.java | 16 +-
.../ozone/om/TestOzoneManagerConfiguration.java | 33 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 35 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 122 +-
.../om/TestOzoneManagerHAWithStoppedNodes.java | 25 +-
.../om/TestOzoneManagerListVolumesSecure.java | 4 +-
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 12 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 20 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 25 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 10 +-
.../ozone/om/multitenant/RangerUserRequest.java | 5 +
.../om/multitenant/TestMultiTenantVolume.java | 10 +-
.../ozone/om/service/TestRangerBGSyncService.java | 6 +-
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 126 +-
.../ozone/om/snapshot/TestOmSnapshotDisabled.java | 6 +-
.../om/snapshot/TestOmSnapshotDisabledRestart.java | 6 +-
.../om/snapshot/TestOmSnapshotFileSystem.java | 64 +-
.../snapshot/TestOmSnapshotFsoWithNativeLib.java | 6 +-
...mSnapshotFsoWithNativeLibWithLinkedBuckets.java | 6 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 23 +-
.../om/snapshot/TestOzoneManagerSnapshotAcl.java | 21 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 8 +-
.../om/snapshot/TestOzoneSnapshotRestore.java | 4 +-
.../snapshot/TestSnapshotBackgroundServices.java | 9 +-
...TestSnapshotDeletingServiceIntegrationTest.java | 55 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 13 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 8 +-
.../hadoop/ozone/recon/TestReconEndpointUtil.java | 2 +-
.../TestReconInsightsForDeletedDirectories.java | 3 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 6 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 16 +-
.../ozone/recon/TestReconWithOzoneManager.java | 25 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 28 +-
.../ozone/reconfig/TestOmReconfiguration.java | 32 +-
.../ozone/reconfig/TestScmReconfiguration.java | 6 +-
.../hadoop/ozone/repair/om/TestFSORepairTool.java | 2 +-
.../ozone/shell/TestDeletedBlocksTxnShell.java | 8 +-
.../shell/TestOzoneContainerUpgradeShell.java | 79 +-
.../hadoop/ozone/shell/TestOzoneDebugShell.java | 142 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 14 +-
.../ozone/shell/TestOzoneShellHAWithFSO.java | 2 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 43 +-
.../hadoop/ozone/shell/TestReconfigShell.java | 187 +-
.../shell/TestReplicationConfigPreference.java | 460 +-
.../tools/contract/AbstractContractDistCpTest.java | 8 +-
.../org/apache/hadoop/utils/FaultInjectorImpl.java | 4 +
.../test/java/org/apache/ozone/test/AclTests.java | 6 +-
.../org/apache/ozone/test/ClusterForTests.java | 6 +
.../java/org/apache/ozone/test/FreonTests.java | 111 +
.../test/java/org/apache/ozone/test/HATests.java | 28 +-
.../java/org/apache/ozone/test/NonHATests.java | 210 +-
...stOzoneIntegrationNonHA.java => TestFreon.java} | 5 +-
.../ozone/test/TestOzoneIntegrationNonHA.java | 1 +
.../src/test/resources/log4j.properties | 6 +
hadoop-ozone/interface-client/pom.xml | 4 +-
.../src/main/proto/OMAdminProtocol.proto | 17 +-
.../src/main/proto/OmClientProtocol.proto | 27 +-
.../interface-client/src/main/resources/proto.lock | 781 ++-
hadoop-ozone/interface-storage/pom.xml | 4 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 12 +-
.../hadoop/ozone/om/helpers/OzoneAclStorage.java | 2 +-
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 5 +-
.../ozone/om/helpers/TestOmPrefixInfoCodec.java | 2 +-
.../ozone/om/helpers/TestTransactionInfoCodec.java | 1 +
.../dev-support/findbugsExcludeFile.xml | 7 +-
.../{ozonefs-common => mini-cluster}/pom.xml | 69 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 80 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 157 +-
.../hadoop/ozone/MiniOzoneClusterProvider.java | 0
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 10 +-
.../hadoop/ozone/UniformDatanodesFactory.java | 13 +-
.../org/apache/hadoop/ozone}/package-info.java | 4 +-
.../dev-support/findbugsExcludeFile.xml | 73 -
hadoop-ozone/ozone-manager/pom.xml | 39 +-
.../hadoop/ozone/om/DeletingServiceMetrics.java | 9 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 5 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 8 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 108 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 112 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 3 +-
.../hadoop/ozone/om/OMMultiTenantManager.java | 1 +
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 11 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 24 +-
.../apache/hadoop/ozone/om/OMStarterInterface.java | 3 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 90 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 2 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 1 -
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 58 +-
.../apache/hadoop/ozone/om/OmSnapshotMetrics.java | 1 -
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 1 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 212 +-
.../hadoop/ozone/om/OzoneManagerPrepareState.java | 8 +-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 11 +-
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 7 -
.../org/apache/hadoop/ozone/om/OzoneTrash.java | 2 +
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 3 +-
.../org/apache/hadoop/ozone/om/ResolvedBucket.java | 1 -
.../hadoop/ozone/om/SnapshotChainManager.java | 2 +-
.../hadoop/ozone/om/SstFilteringService.java | 5 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 12 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 8 +-
.../hadoop/ozone/om/execution/OMExecutionFlow.java | 82 +
.../hadoop/ozone/om/execution}/package-info.java | 4 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 5 +-
.../hadoop/ozone/om/helpers/OMAuditLogger.java | 4 +-
.../apache/hadoop/ozone/om/helpers/OmFSOFile.java | 1 -
.../ozone/om/multitenant/AuthorizerLockImpl.java | 2 +-
.../multitenant/MultiTenantAccessController.java | 18 +
.../RangerClientMultiTenantAccessController.java | 4 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 5 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 4 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 9 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 18 +-
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 38 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 4 +-
.../om/request/bucket/OMBucketCreateRequest.java | 2 +-
.../om/request/bucket/OMBucketSetOwnerRequest.java | 2 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 1 -
.../om/request/file/OMDirectoryCreateRequest.java | 2 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 2 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 4 +-
.../request/file/OMFileCreateRequestWithFSO.java | 2 +-
.../ozone/om/request/file/OMFileRequest.java | 2 +-
.../om/request/file/OMRecoverLeaseRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 2 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 2 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 15 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 3 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 6 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 2 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 2 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 4 +
.../request/key/acl/prefix/OMPrefixAclRequest.java | 2 +-
.../S3MultipartUploadCompleteRequest.java | 2 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 6 -
.../tenant/OMSetRangerServiceVersionRequest.java | 4 -
.../s3/tenant/OMTenantAssignAdminRequest.java | 2 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 2 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 2 +-
.../request/s3/tenant/OMTenantDeleteRequest.java | 4 +-
.../s3/tenant/OMTenantRevokeAdminRequest.java | 2 +-
.../tenant/OMTenantRevokeUserAccessIdRequest.java | 4 +-
.../snapshot/OMSnapshotMoveDeletedKeysRequest.java | 5 -
.../snapshot/OMSnapshotMoveTableKeysRequest.java | 4 -
.../request/snapshot/OMSnapshotRenameRequest.java | 1 -
.../request/upgrade/OMFinalizeUpgradeRequest.java | 2 +-
.../ozone/om/request/upgrade/OMPrepareRequest.java | 5 +-
.../om/request/util/OMMultipartUploadUtils.java | 6 +-
.../ozone/om/request/util/OmKeyHSyncUtil.java | 2 +-
.../om/request/validation/VersionExtractor.java | 1 +
.../om/request/volume/OMVolumeCreateRequest.java | 6 -
.../ozone/om/request/volume/OMVolumeRequest.java | 6 -
.../hadoop/ozone/om/response/OMClientResponse.java | 1 -
.../response/file/OMDirectoryCreateResponse.java | 2 +-
.../file/OMDirectoryCreateResponseWithFSO.java | 2 +-
.../response/key/AbstractOMKeyDeleteResponse.java | 1 -
.../ozone/om/response/key/OMKeyCreateResponse.java | 2 +-
.../key/acl/prefix/OMPrefixAclResponse.java | 2 +-
.../response/s3/tenant/OMTenantDeleteResponse.java | 2 +-
.../tenant/OMTenantRevokeUserAccessIdResponse.java | 2 +-
.../upgrade/OMFinalizeUpgradeResponse.java | 1 +
.../om/response/volume/OMVolumeDeleteResponse.java | 2 +-
.../response/volume/OMVolumeSetOwnerResponse.java | 2 +-
.../om/service/AbstractKeyDeletingService.java | 45 +-
.../hadoop/ozone/om/service/CompactDBService.java | 70 +
.../hadoop/ozone/om/service/CompactionService.java | 184 +
.../ozone/om/service/DirectoryDeletingService.java | 2 +-
.../ozone/om/service/KeyDeletingService.java | 2 +-
.../ozone/om/service/OMRangerBGSyncService.java | 2 +-
.../ozone/om/service/OpenKeyCleanupService.java | 11 +-
.../hadoop/ozone/om/service/QuotaRepairTask.java | 11 +-
.../ozone/om/service/SnapshotDeletingService.java | 27 +-
.../om/service/SnapshotDiffCleanupService.java | 2 -
.../service/SnapshotDirectoryCleaningService.java | 3 +-
.../om/snapshot/FSODirectoryPathResolver.java | 6 +-
.../ozone/om/snapshot/MultiSnapshotLocks.java | 85 +
.../ozone/om/snapshot/RocksDbPersistentSet.java | 1 -
.../ozone/om/snapshot/SnapshotDiffManager.java | 157 +-
.../hadoop/ozone/om/snapshot/SnapshotUtils.java | 50 +-
.../ozone/om/upgrade/OMLayoutVersionManager.java | 7 -
.../ozone/om/upgrade/QuotaRepairUpgradeAction.java | 1 +
.../protocolPB/OMAdminProtocolServerSideImpl.java | 27 +
...OzoneManagerProtocolServerSideTranslatorPB.java | 58 +-
.../protocolPB/OzoneManagerRequestHandler.java | 63 +-
.../hadoop/ozone/security/OMCertificateClient.java | 2 +-
.../OzoneDelegationTokenSecretManager.java | 14 +-
.../hadoop/ozone/security/OzoneSecretStore.java | 3 +-
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 10 +-
.../hadoop/ozone/om/TestAuthorizerLockImpl.java | 2 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 5 -
.../apache/hadoop/ozone/om/TestChunkStreams.java | 4 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 214 +
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 4 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 68 +-
.../hadoop/ozone/om/TestOmSnapshotManager.java | 22 +-
.../ozone/om/TestOzoneManagerHttpServer.java | 12 +-
.../org/apache/hadoop/ozone/om/TestScmClient.java | 7 +-
.../hadoop/ozone/om/failover/TestOMFailovers.java | 6 +-
.../apache/hadoop/ozone/om/ha/TestOMHAMetrics.java | 4 +-
.../ozone/om/lock/TestOzoneLockProvider.java | 1 +
....java => MultiTenantAccessControllerTests.java} | 73 +-
.../TestInMemoryMultiTenantAccessController.java | 32 +-
...estRangerClientMultiTenantAccessController.java | 86 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 18 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 3 +
.../om/ratis/TestOzoneManagerRatisServer.java | 10 +-
.../TestOmRatisSnapshotProvider.java | 21 +-
.../ozone/om/request/OMRequestTestUtils.java | 5 +-
.../ozone/om/request/TestNormalizePaths.java | 5 +-
.../request/TestOMClientRequestWithUserInfo.java | 3 +-
.../request/bucket/TestOMBucketCreateRequest.java | 2 +-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 1 +
.../bucket/TestOMBucketSetPropertyRequest.java | 8 +-
.../request/file/TestOMDirectoryCreateRequest.java | 8 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 25 +-
.../file/TestOMFileCreateRequestWithFSO.java | 14 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 1 -
.../TestOMDirectoriesPurgeRequestAndResponse.java | 1 +
.../ozone/om/request/key/TestOMKeyAclRequest.java | 18 +-
.../om/request/key/TestOMKeyAclRequestWithFSO.java | 6 +-
.../om/request/key/TestOMKeyCommitRequest.java | 9 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 3 +
.../om/request/key/TestOMKeyCreateRequest.java | 6 -
.../request/key/TestOMKeyRenameRequestWithFSO.java | 1 +
.../ozone/om/request/key/TestOMKeyRequest.java | 17 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 1 -
.../request/key/TestOMSetTimesRequestWithFSO.java | 3 +
.../TestS3ExpiredMultipartUploadsAbortRequest.java | 2 -
.../TestS3InitiateMultipartUploadRequest.java | 2 -
...estS3InitiateMultipartUploadRequestWithFSO.java | 11 +-
.../s3/multipart/TestS3MultipartRequest.java | 2 -
.../TestS3MultipartUploadAbortRequest.java | 2 -
.../TestS3MultipartUploadCommitPartRequest.java | 1 -
.../s3/security/TestS3GetSecretRequest.java | 2 -
.../s3/tagging/TestS3PutObjectTaggingRequest.java | 1 -
.../TestS3PutObjectTaggingRequestWithFSO.java | 2 -
.../security/TestOMDelegationTokenRequest.java | 2 -
.../volume/TestOMVolumeSetOwnerRequest.java | 3 -
.../volume/TestOMVolumeSetQuotaRequest.java | 8 +-
.../ozone/om/response/TestCleanupTableInfo.java | 1 -
.../bucket/TestOMBucketCreateResponse.java | 2 -
.../bucket/TestOMBucketDeleteResponse.java | 1 -
.../bucket/TestOMBucketSetPropertyResponse.java | 3 -
.../file/TestOMDirectoryCreateResponseWithFSO.java | 1 +
.../om/response/key/TestOMKeyDeleteResponse.java | 2 +-
.../om/response/key/TestOMKeyRenameResponse.java | 1 +
.../key/TestOMKeyRenameResponseWithFSO.java | 2 +
.../om/response/key/TestOMKeysDeleteResponse.java | 1 -
.../key/TestOMKeysDeleteResponseWithFSO.java | 1 +
.../key/acl/prefix/TestOMPrefixAclResponse.java | 4 +-
...TestS3ExpiredMultipartUploadsAbortResponse.java | 1 -
...stS3InitiateMultipartUploadResponseWithFSO.java | 2 +
.../s3/multipart/TestS3MultipartResponse.java | 2 -
.../TestS3MultipartUploadAbortResponse.java | 1 -
...stS3MultipartUploadCompleteResponseWithFSO.java | 2 -
.../snapshot/TestOMSnapshotCreateResponse.java | 4 +-
.../om/response/volume/TestOMVolumeResponse.java | 1 +
.../ozone/om/service/TestCompactionService.java | 178 +
.../ozone/om/service/TestKeyDeletingService.java | 2 +-
.../service/TestMultipartUploadCleanupService.java | 6 +-
.../om/service/TestOpenKeyCleanupService.java | 11 +-
.../om/service/TestSnapshotDeletingService.java | 3 +-
.../om/service/TestSnapshotDiffCleanupService.java | 14 +-
.../om/snapshot/TestFSODirectoryPathResolver.java | 26 +-
.../ozone/om/snapshot/TestMultiSnapshotLocks.java | 136 +
.../ozone/om/snapshot/TestOmSnapshotUtils.java | 1 +
.../om/snapshot/TestRocksDbPersistentMap.java | 6 +-
.../ozone/om/snapshot/TestSnapshotCache.java | 2 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 142 +-
.../ozone/om/snapshot/TestSstFilteringService.java | 7 +-
.../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 18 +-
.../om/upgrade/TestOzoneManagerPrepareState.java | 11 +-
.../protocolPB/TestOzoneManagerRequestHandler.java | 1 -
.../TestOzoneDelegationTokenSecretManager.java | 6 +-
.../ozone/security/TestOzoneTokenIdentifier.java | 51 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 25 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 15 +-
hadoop-ozone/ozonefs-common/pom.xml | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 3 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 10 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 7 +-
.../fs/ozone/CapableOzoneFSDataStreamOutput.java | 1 +
.../fs/ozone/CapableOzoneFSOutputStream.java | 1 +
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 1 -
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 4 +-
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 2 -
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 2 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 9 +-
hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 258 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 1 +
hadoop-ozone/pom.xml | 18 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
.../ozone/recon/codegen/JooqCodeGenerator.java | 18 +-
.../ozone/recon/codegen/TableNamingStrategy.java | 2 +-
.../ozone/recon/codegen/package-info.java | 2 +-
.../recon/schema/ContainerSchemaDefinition.java | 9 +-
.../ozone/recon/schema/ReconSchemaDefinition.java | 2 +-
.../recon/schema}/ReconSchemaGenerationModule.java | 11 +-
.../ozone/recon/schema}/ReconSqlDbConfig.java | 2 +-
.../recon/schema/ReconTaskSchemaDefinition.java | 4 +-
.../recon/schema/SchemaVersionTableDefinition.java | 6 +-
.../ozone/recon/schema}/SqlDbUtils.java | 2 +-
.../ozone/recon/schema/StatsSchemaDefinition.java | 4 +-
.../recon/schema/UtilizationSchemaDefinition.java | 4 +-
.../ozone/recon/schema/package-info.java | 2 +-
.../recon/dev-support/findbugsExcludeFile.xml | 12 +-
hadoop-ozone/recon/pom.xml | 5 +-
.../apache/hadoop/ozone/recon/ReconConstants.java | 18 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 34 +-
.../hadoop/ozone/recon/ReconSchemaManager.java | 4 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 2 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 2 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 139 +-
.../apache/hadoop/ozone/recon/TarExtractor.java | 186 +
.../ozone/recon/api/ClusterStateEndpoint.java | 10 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 232 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 9 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 5 +-
.../ozone/recon/api/OMDBInsightEndpoint.java | 10 +-
.../hadoop/ozone/recon/api/TaskStatusService.java | 4 +-
.../ozone/recon/api/UtilizationEndpoint.java | 15 +-
.../ozone/recon/api/handlers/BucketHandler.java | 15 +-
.../ozone/recon/api/handlers/FSOBucketHandler.java | 9 +-
.../recon/api/handlers/LegacyBucketHandler.java | 10 +-
.../ozone/recon/api/handlers/OBSBucketHandler.java | 5 +-
.../ozone/recon/api/types/DatanodeMetadata.java | 1 +
.../ozone/recon/api/types/DatanodesResponse.java | 1 -
.../hadoop/ozone/recon/api/types/KeysResponse.java | 1 +
.../ozone/recon/api/types/ListKeysResponse.java | 1 -
.../recon/api/types/NamespaceSummaryResponse.java | 1 -
.../hadoop/ozone/recon/api/types/ObjectDBInfo.java | 1 -
.../ozone/recon/api/types/PipelineMetadata.java | 1 -
.../ozone/recon/api/types/PipelinesResponse.java | 1 -
.../api/types/UnhealthyContainerMetadata.java | 2 +-
.../api/types/UnhealthyContainersResponse.java | 2 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 166 +-
.../ozone/recon/fsck/ReconSafeModeMgrTask.java | 6 +-
.../ozone/recon/heatmap/HeatMapServiceImpl.java | 2 +-
.../hadoop/ozone/recon/heatmap/HeatMapUtil.java | 19 +-
.../ozone/recon/heatmap/IHeatMapProvider.java | 2 +-
.../recon/metrics/ContainerHealthMetrics.java | 86 +
.../recon/metrics/ReconTaskStatusMetrics.java | 4 +-
.../persistence/ContainerHealthSchemaManager.java | 51 +-
.../recon/persistence/DerbyDataSourceProvider.java | 4 +-
.../recon/scm/ReconContainerReportHandler.java | 9 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 8 +-
.../ReconIncrementalContainerReportHandler.java | 60 +-
.../ozone/recon/scm/ReconNewNodeHandler.java | 3 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 8 +-
.../ozone/recon/scm/ReconStaleNodeHandler.java | 4 +-
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 2 -
.../scm/ReconStorageContainerManagerFacade.java | 15 +-
.../recon/spi/ReconContainerMetadataManager.java | 5 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 172 +-
.../impl/ReconContainerMetadataManagerImpl.java | 128 +-
.../impl/StorageContainerServiceProviderImpl.java | 111 +-
...pperTask.java => ContainerKeyMapperHelper.java} | 518 +-
.../recon/tasks/ContainerKeyMapperTaskFSO.java | 65 +
.../recon/tasks/ContainerKeyMapperTaskOBS.java | 64 +
.../ozone/recon/tasks/ContainerSizeCountTask.java | 17 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 333 --
.../ozone/recon/tasks/FileSizeCountTaskFSO.java | 70 +
.../ozone/recon/tasks/FileSizeCountTaskHelper.java | 337 ++
.../ozone/recon/tasks/FileSizeCountTaskOBS.java | 70 +
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 104 +-
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 33 +-
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 50 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 43 +-
.../ozone/recon/tasks/NSSummaryTaskWithOBS.java | 47 +-
.../hadoop/ozone/recon/tasks/OmTableHandler.java | 1 -
.../ozone/recon/tasks/OmTableInsightTask.java | 32 +-
.../hadoop/ozone/recon/tasks/ReconOmTask.java | 110 +-
.../ozone/recon/tasks/ReconTaskController.java | 10 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 90 +-
.../tasks/updater/ReconTaskStatusUpdater.java | 12 +-
.../updater/ReconTaskStatusUpdaterManager.java | 4 +-
.../upgrade/InitialConstraintUpgradeAction.java | 6 +-
.../recon/upgrade/ReconLayoutVersionManager.java | 1 -
.../upgrade/ReconTaskStatusTableUpgradeAction.java | 9 +-
.../UnhealthyContainerReplicaMismatchAction.java | 6 +-
.../webapps/recon/ozone-recon-web/package.json | 6 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 612 +-
.../src/__tests__/datanodes/Datanodes.test.tsx | 19 +-
.../__tests__/datanodes/DatanodesTable.test.tsx | 15 +-
.../src/__tests__/locators/locators.ts | 6 +
.../mocks/pipelineMocks/pipelineResponseMocks.ts | 532 ++
.../mocks/pipelineMocks/pipelinesServer.ts | 35 +
.../src/__tests__/pipelines/Pipelines.test.tsx | 137 +
.../__tests__/pipelines/PipelinesTable.test.tsx | 186 +
.../src/__tests__/utils/pipelines.utils.tsx | 23 +
.../src/v2/components/duMetadata/duMetadata.tsx | 16 +-
.../src/v2/components/tables/containersTable.tsx | 30 +-
.../src/v2/components/tables/pipelinesTable.tsx | 23 +-
.../src/views/diskUsage/diskUsage.tsx | 31 +-
.../ozone-recon-web/src/views/insights/om/om.tsx | 27 +-
.../views/missingContainers/missingContainers.tsx | 21 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 6 +-
.../hadoop/ozone/recon/ReconTestInjector.java | 1 -
.../apache/hadoop/ozone/recon/TestReconUtils.java | 5 +-
.../ozone/recon/api/TestClusterStateEndpoint.java | 5 +-
.../ozone/recon/api/TestContainerEndpoint.java | 225 +-
.../recon/api/TestDeletedKeysSearchEndpoint.java | 3 -
.../hadoop/ozone/recon/api/TestEndpoints.java | 118 +-
.../recon/api/TestNSSummaryDiskUsageOrdering.java | 3 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 16 +-
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 15 +-
.../api/TestNSSummaryEndpointWithOBSAndLegacy.java | 28 +-
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 33 +-
.../ozone/recon/api/TestOpenContainerCount.java | 2 +-
.../recon/api/TestOpenKeysSearchEndpoint.java | 3 +-
.../ozone/recon/api/TestTaskStatusService.java | 4 +-
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 5 +-
.../hadoop/ozone/recon/common/CommonUtils.java | 2 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 115 +-
.../TestContainerHealthTaskRecordGenerator.java | 12 +-
.../recon/persistence/AbstractReconSqlDBTest.java | 4 +-
.../TestReconInternalSchemaDefinition.java | 6 +-
.../persistence/TestReconWithDifferentSqlDBs.java | 9 +-
.../TestSchemaVersionTableDefinition.java | 13 +-
.../recon/persistence/TestSqlSchemaSetup.java | 4 +-
.../persistence/TestStatsSchemaDefinition.java | 6 +-
.../TestUtilizationSchemaDefinition.java | 22 +-
...TestReconIncrementalContainerReportHandler.java | 4 +-
.../ozone/recon/scm/TestReconNodeManager.java | 32 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 1 -
.../impl/TestOzoneManagerServiceProviderImpl.java | 42 +-
.../TestReconContainerMetadataManagerImpl.java | 14 +-
.../hadoop/ozone/recon/tasks/DummyReconDBTask.java | 16 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 144 +-
.../recon/tasks/TestContainerSizeCountTask.java | 35 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 267 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 3 +-
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 108 +-
.../recon/tasks/TestNSSummaryTaskWithLegacy.java | 13 +-
.../TestNSSummaryTaskWithLegacyOBSLayout.java | 12 +-
.../recon/tasks/TestNSSummaryTaskWithOBS.java | 18 +-
.../ozone/recon/tasks/TestOmTableInsightTask.java | 56 +-
.../recon/tasks/TestReconTaskControllerImpl.java | 21 +-
.../recon/tasks/TestReconTaskStatusUpdater.java | 4 +-
.../TestInitialConstraintUpgradeAction.java | 4 +-
hadoop-ozone/s3-secret-store/pom.xml | 4 +-
.../s3/remote/vault/VaultS3SecretStoreBuilder.java | 1 -
.../ozone/s3/remote/vault/auth/AppRoleAuth.java | 1 -
.../s3gateway/dev-support/findbugsExcludeFile.xml | 7 +-
hadoop-ozone/s3gateway/pom.xml | 4 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 2 -
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 1 -
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 5 -
.../hadoop/ozone/s3/SignedChunksInputStream.java | 35 +-
.../org/apache/hadoop/ozone/s3/TracingFilter.java | 1 -
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 3 -
.../hadoop/ozone/s3/commontypes/package-info.java | 8 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 10 +-
.../endpoint/CompleteMultipartUploadRequest.java | 3 +-
.../endpoint/CompleteMultipartUploadResponse.java | 3 +-
.../ozone/s3/endpoint/CopyObjectResponse.java | 6 +-
.../hadoop/ozone/s3/endpoint/CopyPartResult.java | 3 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 1 -
.../ozone/s3/endpoint/ListBucketResponse.java | 3 +-
.../s3/endpoint/ListMultipartUploadsResult.java | 3 +-
.../ozone/s3/endpoint/ListObjectResponse.java | 4 +-
.../ozone/s3/endpoint/ListPartsResponse.java | 5 +-
.../ozone/s3/endpoint/MultiDeleteRequest.java | 7 +-
.../ozone/s3/endpoint/MultiDeleteResponse.java | 10 +-
.../endpoint/MultipartUploadInitiateResponse.java | 3 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 18 +-
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 8 +-
.../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 3 +-
.../apache/hadoop/ozone/s3/endpoint/S3Tagging.java | 3 +-
.../hadoop/ozone/s3/endpoint/package-info.java | 8 +-
.../s3/exception/BadRequestExceptionMapper.java | 1 +
.../hadoop/ozone/s3/exception/OS3Exception.java | 1 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 1 -
.../ozone/s3/signature/AWSSignatureProcessor.java | 2 +-
.../s3/signature/AuthorizationV4HeaderParser.java | 7 +-
.../s3/signature/AuthorizationV4QueryParser.java | 9 +-
.../hadoop/ozone/s3/signature/Credential.java | 4 -
.../hadoop/ozone/s3/signature/SignatureInfo.java | 1 -
.../ozone/s3/signature/StringToSignProducer.java | 22 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 2 +-
.../apache/hadoop/ozone/s3/util/RangeHeader.java | 1 -
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 19 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 19 +
.../hadoop/ozone/client/ClientProtocolStub.java | 13 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 1 +
.../hadoop/ozone/s3/TestAuthorizationFilter.java | 2 +-
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 2 +-
.../s3/endpoint/TestAbortMultipartUpload.java | 1 -
.../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 1 -
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 2 -
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 16 -
...CompleteMultipartUploadRequestUnmarshaller.java | 5 +-
.../hadoop/ozone/s3/endpoint/TestListParts.java | 3 -
.../TestMultiDeleteRequestUnmarshaller.java | 4 +-
.../s3/endpoint/TestMultipartUploadComplete.java | 1 -
.../s3/endpoint/TestMultipartUploadWithCopy.java | 2 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 4 +-
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 10 +-
.../ozone/s3/endpoint/TestObjectTaggingPut.java | 13 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 7 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 6 +-
.../signature/TestAuthorizationV4HeaderParser.java | 1 -
.../s3/signature/TestStringToSignProducer.java | 1 +
hadoop-ozone/tools/pom.xml | 23 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 1 -
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 5 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 5 +-
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 4 +-
.../admin/nssummary/QuotaUsageSubCommand.java | 5 +-
.../ozone/admin/nssummary/SummarySubCommand.java | 5 +-
.../admin/om/FinalizationStatusSubCommand.java | 4 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 24 +-
.../hadoop/ozone/admin/om/PrepareSubCommand.java | 5 +-
.../admin/scm/FinalizationScmStatusSubcommand.java | 4 +-
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 24 +-
.../admin/scm/FinalizeUpgradeCommandUtil.java | 90 -
.../scm/GetFailedDeletedBlocksTxnSubcommand.java | 6 +-
.../admin/scm/GetScmRatisRolesSubcommand.java | 25 +-
.../scm/ResetDeletedBlockRetryCountSubcommand.java | 5 +-
.../parser/ContainerDatanodeDatabase.java | 235 +
.../parser/ContainerLogFileParser.java | 221 +
.../hadoop/ozone/containerlog/parser/DBConsts.java | 32 +-
.../containerlog/parser/DatanodeContainerInfo.java | 137 +
.../ozone/containerlog/parser}/package-info.java | 5 +-
.../{shell/checknative => debug}/CheckNative.java | 15 +-
.../hadoop/ozone/debug/FindMissingPadding.java | 268 -
.../apache/hadoop/ozone/debug/RocksDBUtils.java | 2 -
.../{ => debug}/audit/parser/AuditParser.java | 20 +-
.../audit/parser/common/DatabaseHelper.java | 15 +-
.../audit/parser/common/ParserConsts.java | 2 +-
.../audit/parser/common/package-info.java | 0
.../audit/parser/handler/LoadCommandHandler.java | 8 +-
.../audit/parser/handler/QueryCommandHandler.java | 10 +-
.../parser/handler/TemplateCommandHandler.java | 10 +-
.../audit/parser/handler/package-info.java | 0
.../{ => debug}/audit/parser/model/AuditEntry.java | 2 +-
.../audit/parser/model/package-info.java | 0
.../{ => debug}/audit/parser/package-info.java | 0
.../ContainerLogController.java} | 18 +-
.../ozone/debug/container/ContainerLogParser.java | 93 +
.../ozone/debug/container}/package-info.java | 6 +-
.../datanode/container/ContainerCommands.java | 6 +-
.../debug/datanode/container/ExportSubcommand.java | 6 +-
.../apache/hadoop/ozone/debug/ldb/DBScanner.java | 7 +-
.../apache/hadoop/ozone/debug/ldb/DropTable.java | 74 -
.../apache/hadoop/ozone/debug/ldb/RDBParser.java | 1 -
.../apache/hadoop/ozone/debug/ldb/ValueSchema.java | 2 +-
.../{ReadReplicas.java => replicas/Checksums.java} | 185 +-
.../ozone/debug/replicas/ReplicaVerifier.java | 15 +-
.../hadoop/ozone/debug/replicas/ReplicasDebug.java | 3 +-
.../ozone/debug/replicas/ReplicasVerify.java | 129 +
.../debug/replicas/chunk/ContainerChunkInfo.java | 1 -
.../ozone/freon/AbstractOmBucketReadWriteOps.java | 12 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 22 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 4 +-
.../ozone/freon/ClosedContainerReplicator.java | 8 +-
.../hadoop/ozone/freon/ContentGenerator.java | 2 +-
.../hadoop/ozone/freon/DatanodeBlockPutter.java | 2 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 5 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 4 -
.../ozone/freon/DatanodeSimulationState.java | 3 +-
.../hadoop/ozone/freon/DatanodeSimulator.java | 10 +-
.../freon/FollowerAppendLogEntryGenerator.java | 4 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 2 +-
.../apache/hadoop/ozone/freon/FreonHttpServer.java | 1 -
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 4 +-
.../ozone/freon/HadoopNestedDirGenerator.java | 4 +-
.../apache/hadoop/ozone/freon/HsyncGenerator.java | 1 -
.../hadoop/ozone/freon/KeyGeneratorUtil.java | 1 +
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 4 +-
.../freon/OzoneClientKeyReadWriteListOps.java | 5 +-
.../org/apache/hadoop/ozone/freon/ProgressBar.java | 2 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 20 +-
.../hadoop/ozone/freon/RangeKeysGenerator.java | 3 -
.../hadoop/ozone/freon/S3EntityGenerator.java | 2 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 2 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 4 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 7 +-
.../freon/containergenerator/GeneratorOm.java | 4 +-
.../freon/containergenerator/GeneratorScm.java | 5 +-
.../org/apache/hadoop/ozone/repair/RepairTool.java | 18 +-
.../OMRepair.java => datanode/DatanodeRepair.java} | 18 +-
.../ozone/repair/datanode}/package-info.java | 4 +-
.../schemaupgrade/ContainerUpgradeResult.java | 124 +
.../schemaupgrade/UpgradeContainerSchema.java | 506 ++
.../datanode/schemaupgrade/UpgradeUtils.java | 102 +-
.../schemaupgrade/VolumeUpgradeResult.java | 119 +
.../datanode/schemaupgrade}/package-info.java | 2 +-
.../{om/OMRepair.java => ldb/LDBRepair.java} | 17 +-
.../ozone/repair/ldb/RocksDBManualCompaction.java | 90 +
.../hadoop/ozone/repair/ldb}/package-info.java | 4 +-
.../apache/hadoop/ozone/repair/om/CompactOMDB.java | 78 +
.../hadoop/ozone/repair/om/FSORepairTool.java | 10 +-
.../apache/hadoop/ozone/repair/om/OMRepair.java | 3 +-
.../repair/scm/cert/RecoverSCMCertificate.java | 1 -
.../ozone/shell/checknative/package-info.java | 21 -
.../apache/hadoop/ozone/utils/AutoCompletion.java | 175 +
.../hadoop/ozone/utils/FormattingCLIUtils.java | 2 +
.../resources/container-log-db-queries.properties | 24 +
.../hadoop/ozone/checknative/TestCheckNative.java | 90 -
.../apache/hadoop/ozone/debug/TestCheckNative.java | 76 +
.../{ => debug}/audit/parser/TestAuditParser.java | 13 +-
.../{ => debug}/audit/parser/package-info.java | 0
.../hadoop/ozone/freon/TestContentGenerator.java | 2 +
.../TestGenerateOzoneRequiredConfigurations.java | 4 +-
.../schemaupgrade/TestUpgradeContainerSchema.java | 378 ++
.../ozone/scm/TestGetScmRatisRolesSubcommand.java | 23 +-
pom.xml | 201 +-
1922 files changed, 38739 insertions(+), 26974 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetConstants.java
index 8cf852ccdd,d01c574301..6128e1cce1
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetConstants.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetConstants.java
@@@ -17,7 -17,7 +17,7 @@@
package org.apache.hadoop.hdds.scm.net;
--import org.apache.hadoop.hdds.scm.net.NodeSchema.LayerType;
++import org.apache.hadoop.hdds.scm.net.NodeSchema.LayerType;
import org.apache.hadoop.ozone.util.StringWithByteString;
/**
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java
index 261073123b,0000000000..9511c87635
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java
@@@ -1,409 -1,0 +1,409 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.checksum;
+
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Striped;
+import java.io.File;
- import java.io.FileInputStream;
+import java.io.FileNotFoundException;
- import java.io.FileOutputStream;
+import java.io.IOException;
++import java.io.InputStream;
++import java.io.OutputStream;
+import java.nio.file.Files;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.utils.SimpleStriped;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class coordinates reading and writing Container checksum information for all containers.
+ */
+public class ContainerChecksumTreeManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ContainerChecksumTreeManager.class);
+
+ // Used to coordinate writes to each container's checksum file.
+ // Each container ID is mapped to a stripe.
+ // The file is atomically renamed into place, so readers do not need coordination.
+ private final Striped<Lock> fileLock;
+ private final ContainerMerkleTreeMetrics metrics;
+
+ /**
+ * Creates one instance that should be used to coordinate all container checksum info within a datanode.
+ */
+ public ContainerChecksumTreeManager(ConfigurationSource conf) {
+ fileLock = SimpleStriped.custom(conf.getObject(DatanodeConfiguration.class).getContainerChecksumLockStripes(),
+ () -> new ReentrantLock(true));
+ metrics = ContainerMerkleTreeMetrics.create();
+ }
+
+ public void stop() {
+ ContainerMerkleTreeMetrics.unregister();
+ }
+
+ /**
+ * Writes the specified container merkle tree to the specified container's checksum file.
+ * The data merkle tree within the file is replaced with the {@code tree} parameter, but all other content of the
+ * file remains unchanged.
+ * Concurrent writes to the same file are coordinated internally.
+ */
+ public ContainerProtos.ContainerChecksumInfo writeContainerDataTree(ContainerData data,
+ ContainerMerkleTreeWriter tree)
+ throws IOException {
+ long containerID = data.getContainerID();
+ Lock writeLock = getLock(containerID);
+ writeLock.lock();
+ try {
+ ContainerProtos.ContainerChecksumInfo.Builder checksumInfoBuilder = null;
+ try {
+ // If the file is not present, we will create the data for the first time. This happens under a write lock.
+ checksumInfoBuilder = readBuilder(data)
+ .orElse(ContainerProtos.ContainerChecksumInfo.newBuilder());
+ } catch (IOException ex) {
+ LOG.error("Failed to read container checksum tree file for container {}. Overwriting it with a new instance.",
+ containerID, ex);
+ checksumInfoBuilder = ContainerProtos.ContainerChecksumInfo.newBuilder();
+ }
+
+ ContainerProtos.ContainerChecksumInfo checksumInfo = checksumInfoBuilder
+ .setContainerID(containerID)
+ .setContainerMerkleTree(captureLatencyNs(metrics.getCreateMerkleTreeLatencyNS(), tree::toProto))
+ .build();
+ write(data, checksumInfo);
+ LOG.debug("Data merkle tree for container {} updated", containerID);
+ return checksumInfo;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Adds the specified blocks to the list of deleted blocks specified in the container's checksum file.
+ * All other content of the file remains unchanged.
+ * Concurrent writes to the same file are coordinated internally.
+ */
+ public void markBlocksAsDeleted(KeyValueContainerData data, Collection<Long> deletedBlockIDs) throws IOException {
+ long containerID = data.getContainerID();
+ Lock writeLock = getLock(containerID);
+ writeLock.lock();
+ try {
+ ContainerProtos.ContainerChecksumInfo.Builder checksumInfoBuilder = null;
+ try {
+ // If the file is not present, we will create the data for the first time. This happens under a write lock.
+ checksumInfoBuilder = readBuilder(data)
+ .orElse(ContainerProtos.ContainerChecksumInfo.newBuilder());
+ } catch (IOException ex) {
+ LOG.error("Failed to read container checksum tree file for container {}. Overwriting it with a new instance.",
+ data.getContainerID(), ex);
+ checksumInfoBuilder = ContainerProtos.ContainerChecksumInfo.newBuilder();
+ }
+
+ // Although the persisted block list should already be sorted, we will sort it here to make sure.
+ // This will automatically fix any bugs in the persisted order that may show up.
+ SortedSet<Long> sortedDeletedBlockIDs = new TreeSet<>(checksumInfoBuilder.getDeletedBlocksList());
+ sortedDeletedBlockIDs.addAll(deletedBlockIDs);
+
+ checksumInfoBuilder
+ .setContainerID(containerID)
+ .clearDeletedBlocks()
+ .addAllDeletedBlocks(sortedDeletedBlockIDs);
+ write(data, checksumInfoBuilder.build());
+ LOG.debug("Deleted block list for container {} updated with {} new blocks", data.getContainerID(),
+ sortedDeletedBlockIDs.size());
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Compares the checksum info of the container with the peer's checksum info and returns a report of the differences.
+ * @param thisChecksumInfo The checksum info of the container on this datanode.
+ * @param peerChecksumInfo The checksum info of the container on the peer datanode.
+ */
+ public ContainerDiffReport diff(ContainerProtos.ContainerChecksumInfo thisChecksumInfo,
+ ContainerProtos.ContainerChecksumInfo peerChecksumInfo) throws
+ StorageContainerException {
+
+ ContainerDiffReport report = new ContainerDiffReport();
+ try {
+ captureLatencyNs(metrics.getMerkleTreeDiffLatencyNS(), () -> {
+ Preconditions.assertNotNull(thisChecksumInfo, "Datanode's checksum info is null.");
+ Preconditions.assertNotNull(peerChecksumInfo, "Peer checksum info is null.");
+ if (thisChecksumInfo.getContainerID() != peerChecksumInfo.getContainerID()) {
+ throw new StorageContainerException("Container ID does not match. Local container ID "
+ + thisChecksumInfo.getContainerID() + " , Peer container ID " + peerChecksumInfo.getContainerID(),
+ ContainerProtos.Result.CONTAINER_ID_MISMATCH);
+ }
+
+ compareContainerMerkleTree(thisChecksumInfo, peerChecksumInfo, report);
+ });
+ } catch (IOException ex) {
+ metrics.incrementMerkleTreeDiffFailures();
+ throw new StorageContainerException("Container Diff failed for container #" + thisChecksumInfo.getContainerID(),
+ ex, ContainerProtos.Result.IO_EXCEPTION);
+ }
+
+ // Update Container Diff metrics based on the diff report.
+ if (report.needsRepair()) {
+ metrics.incrementRepairContainerDiffs();
+ return report;
+ }
+ metrics.incrementNoRepairContainerDiffs();
+ return report;
+ }
+
+ private void compareContainerMerkleTree(ContainerProtos.ContainerChecksumInfo thisChecksumInfo,
+ ContainerProtos.ContainerChecksumInfo peerChecksumInfo,
+ ContainerDiffReport report) {
+ ContainerProtos.ContainerMerkleTree thisMerkleTree = thisChecksumInfo.getContainerMerkleTree();
+ ContainerProtos.ContainerMerkleTree peerMerkleTree = peerChecksumInfo.getContainerMerkleTree();
+ Set<Long> thisDeletedBlockSet = new HashSet<>(thisChecksumInfo.getDeletedBlocksList());
+ Set<Long> peerDeletedBlockSet = new HashSet<>(peerChecksumInfo.getDeletedBlocksList());
+
+ if (thisMerkleTree.getDataChecksum() == peerMerkleTree.getDataChecksum()) {
+ return;
+ }
+
+ List<ContainerProtos.BlockMerkleTree> thisBlockMerkleTreeList = thisMerkleTree.getBlockMerkleTreeList();
+ List<ContainerProtos.BlockMerkleTree> peerBlockMerkleTreeList = peerMerkleTree.getBlockMerkleTreeList();
+ int thisIdx = 0, peerIdx = 0;
+
+ // Step 1: Process both lists while elements are present in both
+ while (thisIdx < thisBlockMerkleTreeList.size() && peerIdx < peerBlockMerkleTreeList.size()) {
+ ContainerProtos.BlockMerkleTree thisBlockMerkleTree = thisBlockMerkleTreeList.get(thisIdx);
+ ContainerProtos.BlockMerkleTree peerBlockMerkleTree = peerBlockMerkleTreeList.get(peerIdx);
+
+ if (thisBlockMerkleTree.getBlockID() == peerBlockMerkleTree.getBlockID()) {
+ // Matching block ID; check if the block is deleted and handle the cases;
+ // 1) If the block is deleted in both the block merkle tree, We can ignore comparing them.
+ // 2) If the block is only deleted in our merkle tree, The BG service should have deleted our
+ // block and the peer's BG service hasn't run yet. We can ignore comparing them.
+ // 3) If the block is only deleted in peer merkle tree, we can't reconcile for this block. It might be
+ // deleted by peer's BG service. We can ignore comparing them.
+ // TODO: HDDS-11765 - Handle missed block deletions from the deleted block ids.
+ if (!thisDeletedBlockSet.contains(thisBlockMerkleTree.getBlockID()) &&
+ !peerDeletedBlockSet.contains(thisBlockMerkleTree.getBlockID()) &&
+ thisBlockMerkleTree.getDataChecksum() != peerBlockMerkleTree.getDataChecksum()) {
+ compareBlockMerkleTree(thisBlockMerkleTree, peerBlockMerkleTree, report);
+ }
+ thisIdx++;
+ peerIdx++;
+ } else if (thisBlockMerkleTree.getBlockID() < peerBlockMerkleTree.getBlockID()) {
+ // this block merkle tree's block id is smaller. Which means our merkle tree has some blocks which the peer
+ // doesn't have. We can skip these, the peer will pick up these block when it reconciles with our merkle tree.
+ thisIdx++;
+ } else {
+ // Peer block's ID is smaller; record missing block if peerDeletedBlockSet doesn't contain the blockId
+ // and advance peerIdx
+ if (!peerDeletedBlockSet.contains(peerBlockMerkleTree.getBlockID())) {
+ report.addMissingBlock(peerBlockMerkleTree);
+ }
+ peerIdx++;
+ }
+ }
+
+ // Step 2: Process remaining blocks in the peer list
+ while (peerIdx < peerBlockMerkleTreeList.size()) {
+ ContainerProtos.BlockMerkleTree peerBlockMerkleTree = peerBlockMerkleTreeList.get(peerIdx);
+ if (!peerDeletedBlockSet.contains(peerBlockMerkleTree.getBlockID())) {
+ report.addMissingBlock(peerBlockMerkleTree);
+ }
+ peerIdx++;
+ }
+
+ // If we have remaining block in thisMerkleTree, we can skip these blocks. The peers will pick this block from
+ // us when they reconcile.
+ }
+
+ private void compareBlockMerkleTree(ContainerProtos.BlockMerkleTree thisBlockMerkleTree,
+ ContainerProtos.BlockMerkleTree peerBlockMerkleTree,
+ ContainerDiffReport report) {
+
+ List<ContainerProtos.ChunkMerkleTree> thisChunkMerkleTreeList = thisBlockMerkleTree.getChunkMerkleTreeList();
+ List<ContainerProtos.ChunkMerkleTree> peerChunkMerkleTreeList = peerBlockMerkleTree.getChunkMerkleTreeList();
+ int thisIdx = 0, peerIdx = 0;
+
+ // Step 1: Process both lists while elements are present in both
+ while (thisIdx < thisChunkMerkleTreeList.size() && peerIdx < peerChunkMerkleTreeList.size()) {
+ ContainerProtos.ChunkMerkleTree thisChunkMerkleTree = thisChunkMerkleTreeList.get(thisIdx);
+ ContainerProtos.ChunkMerkleTree peerChunkMerkleTree = peerChunkMerkleTreeList.get(peerIdx);
+
+ if (thisChunkMerkleTree.getOffset() == peerChunkMerkleTree.getOffset()) {
+ // Possible state when this Checksum != peer Checksum:
+ // thisTree = Healthy, peerTree = Healthy -> Both are healthy, No repair needed. Skip.
+ // thisTree = Unhealthy, peerTree = Healthy -> Add to corrupt chunk.
+ // thisTree = Healthy, peerTree = unhealthy -> Do nothing as thisTree is healthy.
+ // thisTree = Unhealthy, peerTree = Unhealthy -> Do Nothing as both are corrupt.
+ if (thisChunkMerkleTree.getDataChecksum() != peerChunkMerkleTree.getDataChecksum() &&
+ !thisChunkMerkleTree.getIsHealthy() && peerChunkMerkleTree.getIsHealthy()) {
+ report.addCorruptChunk(peerBlockMerkleTree.getBlockID(), peerChunkMerkleTree);
+ }
+ thisIdx++;
+ peerIdx++;
+ } else if (thisChunkMerkleTree.getOffset() < peerChunkMerkleTree.getOffset()) {
+ // this chunk merkle tree's offset is smaller. Which means our merkle tree has some chunks which the peer
+ // doesn't have. We can skip these, the peer will pick up these chunks when it reconciles with our merkle tree.
+ thisIdx++;
+ } else {
+ // Peer chunk's offset is smaller; record missing chunk and advance peerIdx
+ report.addMissingChunk(peerBlockMerkleTree.getBlockID(), peerChunkMerkleTree);
+ peerIdx++;
+ }
+ }
+
+ // Step 2: Process remaining chunks in the peer list
+ while (peerIdx < peerChunkMerkleTreeList.size()) {
+ report.addMissingChunk(peerBlockMerkleTree.getBlockID(), peerChunkMerkleTreeList.get(peerIdx));
+ peerIdx++;
+ }
+
+ // If we have remaining chunks in thisBlockMerkleTree, we can skip these chunks. The peers will pick these
+ // chunks from us when they reconcile.
+ }
+
+ /**
+ * Returns the container checksum tree file for the specified container without deserializing it.
+ */
+ @VisibleForTesting
+ public static File getContainerChecksumFile(ContainerData data) {
+ return new File(data.getMetadataPath(), data.getContainerID() + ".tree");
+ }
+
+ @VisibleForTesting
+ public static File getTmpContainerChecksumFile(ContainerData data) {
+ return new File(data.getMetadataPath(), data.getContainerID() + ".tree.tmp");
+ }
+
+ private Lock getLock(long containerID) {
+ return fileLock.get(containerID);
+ }
+
+ /**
+ * Callers are not required to hold a lock while calling this since writes are done to a tmp file and atomically
+ * swapped into place.
+ */
+ public Optional<ContainerProtos.ContainerChecksumInfo> read(ContainerData data) throws IOException {
+ try {
+ return captureLatencyNs(metrics.getReadContainerMerkleTreeLatencyNS(), () -> readChecksumInfo(data));
+ } catch (IOException ex) {
+ metrics.incrementMerkleTreeReadFailures();
+ throw new IOException(ex);
+ }
+ }
+
+ private Optional<ContainerProtos.ContainerChecksumInfo.Builder> readBuilder(ContainerData data) throws IOException {
+ Optional<ContainerProtos.ContainerChecksumInfo> checksumInfo = read(data);
+ return checksumInfo.map(ContainerProtos.ContainerChecksumInfo::toBuilder);
+ }
+
+ /**
+ * Callers should have acquired the write lock before calling this method.
+ */
+ private void write(ContainerData data, ContainerProtos.ContainerChecksumInfo checksumInfo) throws IOException {
+ // Make sure callers filled in required fields before writing.
+ Preconditions.assertTrue(checksumInfo.hasContainerID());
+
+ File checksumFile = getContainerChecksumFile(data);
+ File tmpChecksumFile = getTmpContainerChecksumFile(data);
+
- try (FileOutputStream tmpOutputStream = new FileOutputStream(tmpChecksumFile)) {
++ try (OutputStream tmpOutputStream = Files.newOutputStream(tmpChecksumFile.toPath())) {
+ // Write to a tmp file and rename it into place.
+ captureLatencyNs(metrics.getWriteContainerMerkleTreeLatencyNS(), () -> {
+ checksumInfo.writeTo(tmpOutputStream);
+ Files.move(tmpChecksumFile.toPath(), checksumFile.toPath(), ATOMIC_MOVE);
+ });
+ } catch (IOException ex) {
+ // If the move failed and left behind the tmp file, the tmp file will be overwritten on the next successful write.
+ // Nothing reads directly from the tmp file.
+ metrics.incrementMerkleTreeWriteFailures();
+ throw new IOException("Error occurred when writing container merkle tree for containerID "
+ + data.getContainerID(), ex);
+ }
+ // Set in-memory data checksum.
+ data.setDataChecksum(checksumInfo.getContainerMerkleTree().getDataChecksum());
+ }
+
+ /**
+ * Reads the container checksum info file from the disk as bytes.
+ * Callers are not required to hold a lock while calling this since writes are done to a tmp file and atomically
+ * swapped into place.
+ *
+ * @throws FileNotFoundException When the file does not exist. It may not have been generated yet for this container.
+ * @throws IOException On error reading the file.
+ */
+ public ByteString getContainerChecksumInfo(KeyValueContainerData data) throws IOException {
+ File checksumFile = getContainerChecksumFile(data);
- try (FileInputStream inStream = new FileInputStream(checksumFile)) {
++ try (InputStream inStream = Files.newInputStream(checksumFile.toPath())) {
+ return ByteString.readFrom(inStream);
+ }
+ }
+
+ /**
+ * Reads the container checksum info file (containerID.tree) from the disk.
+ * Callers are not required to hold a lock while calling this since writes are done to a tmp file and atomically
+ * swapped into place.
+ */
+ public static Optional<ContainerProtos.ContainerChecksumInfo> readChecksumInfo(ContainerData data)
+ throws IOException {
+ long containerID = data.getContainerID();
+ File checksumFile = getContainerChecksumFile(data);
+ try {
+ if (!checksumFile.exists()) {
+ LOG.debug("No checksum file currently exists for container {} at the path {}", containerID, checksumFile);
+ return Optional.empty();
+ }
- try (FileInputStream inStream = new FileInputStream(checksumFile)) {
++ try (InputStream inStream = Files.newInputStream(checksumFile.toPath())) {
+ return Optional.of(ContainerProtos.ContainerChecksumInfo.parseFrom(inStream));
+ }
+ } catch (IOException ex) {
+ throw new IOException("Error occurred when reading container merkle tree for containerID "
+ + data.getContainerID() + " at path " + checksumFile, ex);
+ }
+ }
+
+ @VisibleForTesting
+ public ContainerMerkleTreeMetrics getMetrics() {
+ return this.metrics;
+ }
+
+ public static boolean checksumFileExist(Container container) {
+ File checksumFile = getContainerChecksumFile(container.getContainerData());
+ return checksumFile.exists();
+ }
+
+}
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
index d5ba243dd1,0000000000..8556ce5f6d
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
@@@ -1,138 -1,0 +1,134 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.checksum;
+
+import static org.apache.hadoop.ozone.container.common.helpers.TokenHelper.encode;
+
+import com.google.common.collect.ImmutableList;
+import jakarta.annotation.Nonnull;
+import java.io.IOException;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.hdds.scm.XceiverClientSpi;
+import org.apache.hadoop.hdds.scm.client.ClientTrustManager;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.hdds.security.SecurityConfig;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.container.common.helpers.TokenHelper;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
+
+/**
+ * This class wraps necessary container-level rpc calls for container reconciliation.
+ * - GetContainerMerkleTree
+ */
+public class DNContainerOperationClient implements AutoCloseable {
+
- private static final Logger LOG =
- LoggerFactory.getLogger(DNContainerOperationClient.class);
+ private final TokenHelper tokenHelper;
+ private final XceiverClientManager xceiverClientManager;
+
+ public DNContainerOperationClient(ConfigurationSource conf,
+ CertificateClient certificateClient,
+ SecretKeySignerClient secretKeyClient) throws IOException {
+ this.tokenHelper = new TokenHelper(new SecurityConfig(conf), secretKeyClient);
+ this.xceiverClientManager = createClientManager(conf, certificateClient);
+ }
+
+ @Nonnull
+ private static XceiverClientManager createClientManager(
+ ConfigurationSource conf, CertificateClient certificateClient)
+ throws IOException {
+ ClientTrustManager trustManager = null;
+ if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
+ trustManager = certificateClient.createClientTrustManager();
+ }
+ DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class);
+ return new XceiverClientManager(conf,
+ new XceiverClientManager.XceiverClientManagerConfigBuilder()
+ .setMaxCacheSize(dnConf.getContainerClientCacheSize())
+ .setStaleThresholdMs(dnConf.getContainerClientCacheStaleThreshold())
+ .build(), trustManager);
+ }
+
+ public XceiverClientManager getXceiverClientManager() {
+ return xceiverClientManager;
+ }
+
+ public TokenHelper getTokenHelper() {
+ return tokenHelper;
+ }
+
+ /**
+ * Reads {@link ContainerProtos.ContainerChecksumInfo} for a specified container for the specified datanode.
+ *
+ * @throws IOException For errors communicating with the datanode.
+ * @throws StorageContainerException For errors obtaining the checksum info, including the file being missing or
+ * empty on the datanode, or the datanode not having a replica of the container.
+ * @throws InvalidProtocolBufferException If the file received from the datanode cannot be deserialized.
+ */
+ public ContainerProtos.ContainerChecksumInfo getContainerChecksumInfo(long containerId, DatanodeDetails dn)
+ throws IOException {
+ XceiverClientSpi xceiverClient = this.xceiverClientManager.acquireClient(createSingleNodePipeline(dn));
+ try {
+ String containerToken = encode(tokenHelper.getContainerToken(
+ ContainerID.valueOf(containerId)));
+ ContainerProtos.GetContainerChecksumInfoResponseProto response =
+ ContainerProtocolCalls.getContainerChecksumInfo(xceiverClient,
+ containerId, containerToken);
+ ByteString serializedChecksumInfo = response.getContainerChecksumInfo();
+ // Protobuf will convert an empty ByteString into a default value object. Treat this as an error instead, since
+ // the default value will not represent the state of the container.
+ // The server does not deserialize the file before sending it, so we must check the length on the client.
+ if (serializedChecksumInfo.isEmpty()) {
+ throw new StorageContainerException("Empty Container checksum file for container " + containerId + " received",
+ ContainerProtos.Result.IO_EXCEPTION);
+ } else {
+ return ContainerProtos.ContainerChecksumInfo.parseFrom(serializedChecksumInfo);
+ }
+ } finally {
+ this.xceiverClientManager.releaseClient(xceiverClient, false);
+ }
+ }
+
+ public static Pipeline createSingleNodePipeline(DatanodeDetails dn) {
+ return Pipeline.newBuilder()
+ .setNodes(ImmutableList.of(dn))
+ .setId(PipelineID.valueOf(dn.getUuid()))
+ .setState(Pipeline.PipelineState.CLOSED)
+ .setReplicationConfig(StandaloneReplicationConfig.getInstance(
+ HddsProtos.ReplicationFactor.ONE)).build();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (xceiverClientManager != null) {
+ xceiverClientManager.close();
+ }
+ }
+}
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
index 652a90e978,fcf3edb631..7ad04eb3ec
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
@@@ -56,13 -56,9 +56,12 @@@ public class ContainerMetrics implement
@Metric private MutableCounterLong containerForceDelete;
@Metric private MutableCounterLong numReadStateMachine;
@Metric private MutableCounterLong bytesReadStateMachine;
+ @Metric private MutableCounterLong numContainerReconciledWithoutChanges;
+ @Metric private MutableCounterLong numContainerReconciledWithChanges;
-
private final EnumMap<ContainerProtos.Type, MutableCounterLong> numOpsArray;
private final EnumMap<ContainerProtos.Type, MutableCounterLong> opsBytesArray;
+ private final EnumMap<ContainerProtos.Type, MutableCounterLong> opsForClosedContainer;
private final EnumMap<ContainerProtos.Type, MutableRate> opsLatency;
private final EnumMap<ContainerProtos.Type, MutableQuantiles[]> opsLatQuantiles;
private MetricsRegistry registry = null;
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
index 7a34ec2999,3c12b2f8fc..14d4ec70dd
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java
@@@ -73,44 -69,15 +69,15 @@@ public final class ContainerDataYaml
/**
* Creates a .container file in yaml format.
- *
- * @param containerFile
- * @param containerData
- * @throws IOException
*/
- public static void createContainerFile(ContainerType containerType,
- ContainerData containerData, File containerFile) throws IOException {
- Writer writer = null;
- FileOutputStream out = null;
- try {
- boolean withReplicaIndex =
- containerData instanceof KeyValueContainerData &&
- ((KeyValueContainerData) containerData).getReplicaIndex() > 0;
+ public static void createContainerFile(ContainerData containerData, File containerFile) throws IOException {
+ // Create Yaml for given container type
+ final Yaml yaml = getYamlForContainerType(containerData.getContainerType(), containerData.getReplicaIndex() > 0);
+ // Compute Checksum and update ContainerData
- containerData.computeAndSetChecksum(yaml);
++ containerData.computeAndSetContainerFileChecksum(yaml);
- // Create Yaml for given container type
- Yaml yaml = getYamlForContainerType(containerType, withReplicaIndex);
- // Compute Checksum and update ContainerData
- containerData.computeAndSetContainerFileChecksum(yaml);
-
- // Write the ContainerData with checksum to Yaml file.
- out = new FileOutputStream(
- containerFile);
- writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
- yaml.dump(containerData, writer);
- } finally {
- try {
- if (writer != null) {
- writer.flush();
- // make sure the container metadata is synced to disk.
- out.getFD().sync();
- writer.close();
- }
- } catch (IOException ex) {
- LOG.warn("Error occurred during closing the writer. ContainerID: " +
- containerData.getContainerID());
- }
- IOUtils.closeQuietly(out);
- }
+ // Write the ContainerData with checksum to Yaml file.
+ YamlUtils.dump(yaml, containerData, containerFile, LOG);
}
/**
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 0d113d467d,c4e03e4533..9c8fed3ca5
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@@ -71,9 -70,6 +70,8 @@@ import org.apache.hadoop.ozone.containe
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
- import org.apache.hadoop.ozone.container.common.volume.VolumeUsage;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScanError;
+import org.apache.hadoop.ozone.container.ozoneimpl.DataScanResult;
import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
import org.apache.hadoop.util.Time;
import org.apache.ratis.statemachine.StateMachine;
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 76e3673ce6,2cb757f1a1..8fa7157091
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@@ -21,9 -23,8 +21,10 @@@ import com.google.common.annotations.Vi
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+ import java.time.Clock;
+import java.util.Set;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
@@@ -73,13 -72,14 +74,14 @@@ public abstract class Handler
public static Handler getHandlerForContainerType(
final ContainerType containerType, final ConfigurationSource config,
final String datanodeId, final ContainerSet contSet,
- final VolumeSet volumeSet, final ContainerMetrics metrics,
+ final VolumeSet volumeSet, final VolumeChoosingPolicy volumeChoosingPolicy,
+ final ContainerMetrics metrics,
- IncrementalReportSender<Container> icrSender) {
+ IncrementalReportSender<Container> icrSender, ContainerChecksumTreeManager checksumManager) {
switch (containerType) {
case KeyValueContainer:
return new KeyValueHandler(config,
- datanodeId, contSet, volumeSet, metrics,
- icrSender, checksumManager);
+ datanodeId, contSet, volumeSet, volumeChoosingPolicy, metrics,
- icrSender, Clock.systemUTC());
++ icrSender, Clock.systemUTC(), checksumManager);
default:
throw new IllegalArgumentException("Handler for ContainerType: " +
containerType + "doesn't exist.");
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
index be3361ddc2,371aed1ae0..222d519b9a
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
@@@ -43,40 -48,34 +48,38 @@@ public class DatanodeConfiguration exte
public static final String HDDS_DATANODE_BLOCK_DELETE_THREAD_MAX =
CONFIG_PREFIX + "." + BLOCK_DELETE_THREAD_MAX;
- private static final Logger LOG =
- LoggerFactory.getLogger(DatanodeConfiguration.class);
-
- static final String CONTAINER_DELETE_THREADS_MAX_KEY =
- "hdds.datanode.container.delete.threads.max";
- static final String CONTAINER_CLOSE_THREADS_MAX_KEY =
- "hdds.datanode.container.close.threads.max";
- static final String PERIODIC_DISK_CHECK_INTERVAL_MINUTES_KEY =
- "hdds.datanode.periodic.disk.check.interval.minutes";
- public static final String DISK_CHECK_FILE_SIZE_KEY =
- "hdds.datanode.disk.check.io.file.size";
- public static final String DISK_CHECK_IO_TEST_COUNT_KEY =
- "hdds.datanode.disk.check.io.test.count";
- public static final String DISK_CHECK_IO_FAILURES_TOLERATED_KEY =
- "hdds.datanode.disk.check.io.failures.tolerated";
- public static final String FAILED_DATA_VOLUMES_TOLERATED_KEY =
- "hdds.datanode.failed.data.volumes.tolerated";
- public static final String FAILED_METADATA_VOLUMES_TOLERATED_KEY =
- "hdds.datanode.failed.metadata.volumes.tolerated";
- public static final String FAILED_DB_VOLUMES_TOLERATED_KEY =
- "hdds.datanode.failed.db.volumes.tolerated";
- public static final String DISK_CHECK_MIN_GAP_KEY =
- "hdds.datanode.disk.check.min.gap";
- public static final String DISK_CHECK_TIMEOUT_KEY =
- "hdds.datanode.disk.check.timeout";
-
- public static final String WAIT_ON_ALL_FOLLOWERS =
- "hdds.datanode.wait.on.all.followers";
- public static final String CONTAINER_SCHEMA_V3_ENABLED =
- "hdds.datanode.container.schema.v3.enabled";
+ private static final Logger LOG = LoggerFactory.getLogger(DatanodeConfiguration.class);
+
+ static final String CONTAINER_DELETE_THREADS_MAX_KEY = "hdds.datanode.container.delete.threads.max";
+ static final String CONTAINER_CLOSE_THREADS_MAX_KEY = "hdds.datanode.container.close.threads.max";
+ static final String PERIODIC_DISK_CHECK_INTERVAL_MINUTES_KEY = "hdds.datanode.periodic.disk.check.interval.minutes";
+ public static final String DISK_CHECK_FILE_SIZE_KEY = "hdds.datanode.disk.check.io.file.size";
+ public static final String DISK_CHECK_IO_TEST_COUNT_KEY = "hdds.datanode.disk.check.io.test.count";
+ public static final String DISK_CHECK_IO_FAILURES_TOLERATED_KEY = "hdds.datanode.disk.check.io.failures.tolerated";
+ public static final String FAILED_DATA_VOLUMES_TOLERATED_KEY = "hdds.datanode.failed.data.volumes.tolerated";
+ public static final String FAILED_METADATA_VOLUMES_TOLERATED_KEY = "hdds.datanode.failed.metadata.volumes.tolerated";
+ public static final String FAILED_DB_VOLUMES_TOLERATED_KEY = "hdds.datanode.failed.db.volumes.tolerated";
+ public static final String DISK_CHECK_MIN_GAP_KEY = "hdds.datanode.disk.check.min.gap";
+ public static final String DISK_CHECK_TIMEOUT_KEY = "hdds.datanode.disk.check.timeout";
+
+ // Minimum space should be left on volume.
+ // Ex: If volume has 1000GB and minFreeSpace is configured as 10GB,
+ // In this case when availableSpace is 10GB or below, volume is assumed as full
+ public static final String HDDS_DATANODE_VOLUME_MIN_FREE_SPACE = "hdds.datanode.volume.min.free.space";
+ public static final String HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_DEFAULT = "5GB";
+ // Minimum percent of space should be left on volume.
+ // Ex: If volume has 1000GB and minFreeSpacePercent is configured as 2%,
+ // In this case when availableSpace is 20GB(2% of 1000) or below, volume is assumed as full
+ public static final String HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT =
+ "hdds.datanode.volume.min.free.space.percent";
+ static final byte MIN_FREE_SPACE_UNSET = -1;
+
+ public static final String WAIT_ON_ALL_FOLLOWERS = "hdds.datanode.wait.on.all.followers";
+ public static final String CONTAINER_SCHEMA_V3_ENABLED = "hdds.datanode.container.schema.v3.enabled";
+ public static final String CONTAINER_CHECKSUM_LOCK_STRIPES_KEY = "hdds.datanode.container.checksum.lock.stripes";
+ public static final String CONTAINER_CLIENT_CACHE_SIZE = "hdds.datanode.container.client.cache.size";
+ public static final String CONTAINER_CLIENT_CACHE_STALE_THRESHOLD =
+ "hdds.datanode.container.client.cache.stale.threshold";
static final boolean CHUNK_DATA_VALIDATION_CHECK_DEFAULT = false;
@@@ -100,27 -99,24 +103,29 @@@
static final long ROCKSDB_LOG_MAX_FILE_SIZE_BYTES_DEFAULT = 32 * 1024 * 1024;
static final int ROCKSDB_LOG_MAX_FILE_NUM_DEFAULT = 64;
// one hour
- static final long ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICRO_SECONDS_DEFAULT =
- 1L * 60 * 60 * 1000 * 1000;
+ static final long ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICRO_SECONDS_DEFAULT = 1L * 60 * 60 * 1000 * 1000;
static final int ROCKSDB_MAX_OPEN_FILES_DEFAULT = 1024;
- public static final String ROCKSDB_LOG_MAX_FILE_SIZE_BYTES_KEY =
- "hdds.datanode.rocksdb.log.max-file-size";
- public static final String ROCKSDB_LOG_MAX_FILE_NUM_KEY =
- "hdds.datanode.rocksdb.log.max-file-num";
- public static final String
- ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICRO_SECONDS_KEY =
+ public static final String ROCKSDB_LOG_MAX_FILE_SIZE_BYTES_KEY = "hdds.datanode.rocksdb.log.max-file-size";
+ public static final String ROCKSDB_LOG_MAX_FILE_NUM_KEY = "hdds.datanode.rocksdb.log.max-file-num";
+ public static final String ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICRO_SECONDS_KEY =
"hdds.datanode.rocksdb.delete_obsolete_files_period";
- public static final Boolean OZONE_DATANODE_CHECK_EMPTY_CONTAINER_DIR_ON_DELETE_DEFAULT = false;
+ public static final Boolean
+ OZONE_DATANODE_CHECK_EMPTY_CONTAINER_DIR_ON_DELETE_DEFAULT = false;
+ public static final int CONTAINER_CHECKSUM_LOCK_STRIPES_DEFAULT = 127;
+ public static final int CONTAINER_CLIENT_CACHE_SIZE_DEFAULT = 100;
+ public static final int
+ CONTAINER_CLIENT_CACHE_STALE_THRESHOLD_MILLISECONDS_DEFAULT = 10000;
- private static final long
- AUTO_COMPACTION_SMALL_SST_FILE_INTERVAL_MINUTES_DEFAULT = 120;
- private static final int
- AUTO_COMPACTION_SMALL_SST_FILE_THREADS_DEFAULT = 1;
+ private static final long AUTO_COMPACTION_SMALL_SST_FILE_INTERVAL_MINUTES_DEFAULT = 120;
+ private static final int AUTO_COMPACTION_SMALL_SST_FILE_THREADS_DEFAULT = 1;
+
+ static final int CONTAINER_DELETE_THREADS_DEFAULT = 2;
+ static final int CONTAINER_CLOSE_THREADS_DEFAULT = 3;
+ static final int BLOCK_DELETE_THREADS_DEFAULT = 5;
+
+ public static final String BLOCK_DELETE_COMMAND_WORKER_INTERVAL =
+ "hdds.datanode.block.delete.command.worker.interval";
+ public static final Duration BLOCK_DELETE_COMMAND_WORKER_INTERVAL_DEFAULT = Duration.ofSeconds(2);
/**
* Number of threads per volume that Datanode will use for chunk read.
@@@ -623,11 -545,6 +587,7 @@@
)
private long deleteContainerTimeoutMs = Duration.ofSeconds(60).toMillis();
- public long getDeleteContainerTimeoutMs() {
- return deleteContainerTimeoutMs;
- }
-
+ @SuppressWarnings("checkstyle:MethodLength")
@PostConstruct
public void validate() {
if (containerDeleteThreads < 1) {
@@@ -762,28 -679,89 +722,108 @@@
ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICRO_SECONDS_DEFAULT;
}
+ if (containerChecksumLockStripes < 1) {
+ LOG.warn("{} must be at least 1. Defaulting to {}", CONTAINER_CHECKSUM_LOCK_STRIPES_KEY,
+ CONTAINER_CHECKSUM_LOCK_STRIPES_DEFAULT);
+ containerChecksumLockStripes = CONTAINER_CHECKSUM_LOCK_STRIPES_DEFAULT;
+ }
+
+ if (containerClientCacheSize < 1) {
+ LOG.warn("{} must be at least 1. Defaulting to {}", CONTAINER_CLIENT_CACHE_SIZE,
+ CONTAINER_CLIENT_CACHE_SIZE_DEFAULT);
+ containerClientCacheSize = CONTAINER_CLIENT_CACHE_SIZE_DEFAULT;
+ }
+
+ if (containerClientCacheStaleThreshold < 1) {
+ LOG.warn("{} must be at least 1. Defaulting to {}", CONTAINER_CLIENT_CACHE_STALE_THRESHOLD,
+ CONTAINER_CLIENT_CACHE_STALE_THRESHOLD_MILLISECONDS_DEFAULT);
+ containerClientCacheStaleThreshold =
+ CONTAINER_CLIENT_CACHE_STALE_THRESHOLD_MILLISECONDS_DEFAULT;
+ }
++
+ validateMinFreeSpace();
}
- public void setContainerDeleteThreads(int containerDeleteThreads) {
- this.containerDeleteThreads = containerDeleteThreads;
+ /**
+ * If 'hdds.datanode.volume.min.free.space' is defined,
+ * it will be honored first. If it is not defined and
+ * 'hdds.datanode.volume.min.free.space.percent' is defined, it will honor this
+ * else it will fall back to 'hdds.datanode.volume.min.free.space.default'
+ */
+ private void validateMinFreeSpace() {
+ if (minFreeSpaceRatio > 1) {
+ LOG.warn("{} = {} is invalid, should be between 0 and 1",
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
+ minFreeSpaceRatio);
+
+ minFreeSpaceRatio = MIN_FREE_SPACE_UNSET;
+ }
+
+ final boolean minFreeSpaceConfigured = minFreeSpace >= 0;
+ final boolean minFreeSpaceRatioConfigured = minFreeSpaceRatio >= 0;
+
+ if (minFreeSpaceConfigured && minFreeSpaceRatioConfigured) {
+ // Only one property should be configured.
+ // Since both properties are configured, HDDS_DATANODE_VOLUME_MIN_FREE_SPACE is used to determine minFreeSpace
+ LOG.warn("Only one of {}={} and {}={} should be set. With both set, {} value will be used.",
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE,
+ minFreeSpace,
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
+ minFreeSpaceRatio,
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE);
+
+ minFreeSpaceRatio = MIN_FREE_SPACE_UNSET;
+ }
+
+ if (!minFreeSpaceConfigured && !minFreeSpaceRatioConfigured) {
+ // If both are not configured use defaultFreeSpace
+ minFreeSpaceRatio = MIN_FREE_SPACE_UNSET;
+ minFreeSpace = getDefaultFreeSpace();
+ }
+ }
+
+ public Duration getBlockDeletionInterval() {
+ return blockDeletionInterval;
+ }
+
+ public void setBlockDeletionInterval(Duration duration) {
+ blockDeletionInterval = duration;
+ }
+
+ public Duration getRecoveringContainerScrubInterval() {
+ return recoveringContainerScrubInterval;
+ }
+
+ public void setRecoveringContainerScrubInterval(Duration duration) {
+ recoveringContainerScrubInterval = duration;
+ }
+
+ public int getBlockDeletionLimit() {
+ return blockLimitPerInterval;
+ }
+
+ public void setBlockDeletionLimit(int limit) {
+ this.blockLimitPerInterval = limit;
+ }
+
+ public long getDeleteContainerTimeoutMs() {
+ return deleteContainerTimeoutMs;
+ }
+
+ public Duration getBlockDeletingMaxLockHoldingTime() {
+ return Duration.ofMillis(blockDeletingMaxLockHoldingTime);
+ }
+
+ public void setBlockDeletingMaxLockHoldingTime(Duration maxLockHoldingTime) {
+ blockDeletingMaxLockHoldingTime = maxLockHoldingTime.toMillis();
+ }
+
+ public boolean waitOnAllFollowers() {
+ return waitOnAllFollowers;
+ }
+
+ public void setWaitOnAllFollowers(boolean val) {
+ this.waitOnAllFollowers = val;
}
public int getContainerDeleteThreads() {
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 127588cf06,c9412c5be3..48c981863d
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@@ -48,8 -48,8 +48,9 @@@ import org.apache.hadoop.hdds.utils.IOU
import org.apache.hadoop.hdds.utils.NettyMetrics;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.HddsDatanodeStopService;
+import org.apache.hadoop.ozone.container.checksum.DNContainerOperationClient;
import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
+ import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.report.ReportManager;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CloseContainerCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ClosePipelineCommandHandler;
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 8378ff3ca7,09cadd5d13..182e6dc7d8
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@@ -947,13 -926,9 +921,13 @@@ public class KeyValueContainer implemen
@Override
public boolean shouldScanData() {
- final boolean shouldScan = getContainerState() == CLOSED || getContainerState() == QUASI_CLOSED;
- if (!shouldScan) {
- LOG.debug("Healthy container is not scanned: {}", containerData);
+ boolean shouldScan =
- getContainerState() == ContainerDataProto.State.CLOSED
- || getContainerState() == ContainerDataProto.State.QUASI_CLOSED
- || getContainerState() == ContainerDataProto.State.UNHEALTHY;
++ getContainerState() == CLOSED
++ || getContainerState() == QUASI_CLOSED
++ || getContainerState() == UNHEALTHY;
+ if (!shouldScan && LOG.isDebugEnabled()) {
+ LOG.debug("Container {} in state {} should not have its data scanned.",
+ containerData.getContainerID(), containerData.getState());
}
return shouldScan;
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
index 72ab02af25,033a02a4ea..9c754831aa
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
@@@ -25,10 -26,7 +25,11 @@@ import java.io.FileNotFoundException
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+ import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@@ -119,52 -144,6 +120,52 @@@ public class KeyValueContainerCheck
}
}
+ private List<ContainerScanError> scanMetadata() {
+ List<ContainerScanError> metadataErrors = new ArrayList<>();
+ // Container directory should exist.
+ // If it does not, we cannot continue the scan.
+ File containerDir = new File(metadataPath).getParentFile();
+ if (!containerDir.exists()) {
+ metadataErrors.add(new ContainerScanError(FailureType.MISSING_CONTAINER_DIR,
+ containerDir, new FileNotFoundException("Container directory " + containerDir + " not found.")));
+ return metadataErrors;
+ }
+
+ // Metadata directory within the container directory should exist.
+ // If it does not, no further scanning can be done.
+ File metadataDir = new File(metadataPath);
+ if (!metadataDir.exists()) {
+ metadataErrors.add(new ContainerScanError(FailureType.MISSING_METADATA_DIR, metadataDir,
+ new FileNotFoundException("Metadata directory " + metadataDir + " not found.")));
+ return metadataErrors;
+ }
+
+ // Container file inside the metadata directory should be valid.
+ // If it is not, no further scanning can be done.
+ File containerFile = KeyValueContainer.getContainerFile(metadataPath, containerID);
+ try {
+ loadContainerData(containerFile);
- } catch (FileNotFoundException ex) {
++ } catch (FileNotFoundException | NoSuchFileException ex) {
+ metadataErrors.add(new ContainerScanError(FailureType.MISSING_CONTAINER_FILE, containerFile, ex));
+ return metadataErrors;
+ } catch (IOException ex) {
+ metadataErrors.add(new ContainerScanError(FailureType.CORRUPT_CONTAINER_FILE, containerFile, ex));
+ return metadataErrors;
+ }
+ metadataErrors.addAll(checkContainerFile(containerFile));
+
+ // Chunks directory should exist.
+ // The metadata scan can continue even if this fails, since it does not look at the data inside the chunks
+ // directory.
+ File chunksDir = new File(containerDataFromDisk.getChunksPath());
+ if (!chunksDir.exists()) {
+ metadataErrors.add(new ContainerScanError(FailureType.MISSING_CHUNKS_DIR, chunksDir,
+ new FileNotFoundException("Chunks directory " + chunksDir + " not found.")));
+ }
+
+ return metadataErrors;
+ }
+
/**
* full checks comprise scanning all metadata inside the container.
* Including the KV database. These checks are intrusive, consume more
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 43926ca5e2,408897c6bb..5551b77ae2
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@@ -192,9 -156,8 +192,9 @@@ public class KeyValueHandler extends Ha
ContainerSet contSet,
VolumeSet volSet,
ContainerMetrics metrics,
- IncrementalReportSender<Container> icrSender) {
- this(config, datanodeId, contSet, volSet, null, metrics, icrSender, Clock.systemUTC());
+ IncrementalReportSender<Container> icrSender,
+ ContainerChecksumTreeManager checksumManager) {
- this(config, datanodeId, contSet, volSet, metrics, icrSender, checksumManager, Clock.systemUTC());
++ this(config, datanodeId, contSet, volSet, null, metrics, icrSender, Clock.systemUTC(), checksumManager);
}
@SuppressWarnings("checkstyle:ParameterNumber")
@@@ -202,10 -165,10 +202,11 @@@
String datanodeId,
ContainerSet contSet,
VolumeSet volSet,
+ VolumeChoosingPolicy volumeChoosingPolicy,
ContainerMetrics metrics,
IncrementalReportSender<Container> icrSender,
- ContainerChecksumTreeManager checksumManager,
-- Clock clock) {
++ Clock clock,
++ ContainerChecksumTreeManager checksumManager) {
super(config, datanodeId, contSet, volSet, metrics, icrSender);
this.clock = clock;
blockManager = new BlockManagerImpl(config);
@@@ -213,12 -176,8 +214,9 @@@
DatanodeConfiguration.class).isChunkDataValidationCheck();
chunkManager = ChunkManagerFactory.createChunkManager(config, blockManager,
volSet);
+ this.checksumManager = checksumManager;
- try {
- volumeChoosingPolicy = VolumeChoosingPolicyFactory.getPolicy(conf);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
+ this.volumeChoosingPolicy = volumeChoosingPolicy != null ? volumeChoosingPolicy
+ : VolumeChoosingPolicyFactory.getPolicy(config);
maxContainerSize = (long) config.getStorageSize(
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 370b455e33,fffc17db35..c8cbe23add
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@@ -65,7 -66,7 +66,8 @@@ import org.apache.hadoop.hdds.utils.IOU
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.HddsDatanodeService;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
+ import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@@@ -216,7 -218,7 +222,7 @@@ public class OzoneContainer
Handler.getHandlerForContainerType(
containerType, conf,
context.getParent().getDatanodeDetails().getUuidString(),
- containerSet, volumeSet, metrics, icrSender, checksumTreeManager));
- containerSet, volumeSet, volumeChoosingPolicy, metrics, icrSender));
++ containerSet, volumeSet, volumeChoosingPolicy, metrics, icrSender, checksumTreeManager));
}
SecurityConfig secConf = new SecurityConfig(conf);
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java
index 6305158592,f69516f94e..3c3b8548c6
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java
@@@ -122,12 -119,14 +119,14 @@@ public class ContainerImporter
packer.unpackContainerDescriptor(input);
containerData = getKeyValueContainerData(containerDescriptorYaml);
}
- ContainerUtils.verifyChecksum(containerData, conf);
+ ContainerUtils.verifyContainerFileChecksum(containerData, conf);
containerData.setVolume(targetVolume);
- try (FileInputStream input = new FileInputStream(tarFilePath.toFile())) {
+ try (InputStream input = Files.newInputStream(tarFilePath)) {
Container container = controller.importContainer(
containerData, input, packer);
+ // After container import is successful, increase used space for the volume
+ targetVolume.incrementUsedSpace(container.getContainerData().getBytesUsed());
containerSet.addContainerByOverwriteMissingContainer(container);
}
} finally {
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconcileContainerCommand.java
index bfcca3530c,0000000000..681e302ced
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconcileContainerCommand.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconcileContainerCommand.java
@@@ -1,106 -1,0 +1,105 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.protocol.commands;
+
+import static java.util.Collections.emptySet;
+
+import com.google.common.base.Preconditions;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReconcileContainerCommandProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+
+/**
+ * Asks datanodes to reconcile the specified container with other container replicas.
+ */
+public class ReconcileContainerCommand extends SCMCommand<ReconcileContainerCommandProto> {
+
+ private final Set<DatanodeDetails> peerDatanodes;
+
+ public ReconcileContainerCommand(long containerID, Set<DatanodeDetails> peerDatanodes) {
+ // Container ID serves as command ID, since only one reconciliation should be in progress at a time.
+ super(containerID);
+ this.peerDatanodes = peerDatanodes;
+ }
+
-
+ @Override
+ public SCMCommandProto.Type getType() {
+ return SCMCommandProto.Type.reconcileContainerCommand;
+ }
+
+ @Override
+ public ReconcileContainerCommandProto getProto() {
+ ReconcileContainerCommandProto.Builder builder = ReconcileContainerCommandProto.newBuilder()
+ .setContainerID(getId());
+ for (DatanodeDetails dd : peerDatanodes) {
+ builder.addPeers(dd.getProtoBufMessage());
+ }
+ return builder.build();
+ }
+
+ public Set<DatanodeDetails> getPeerDatanodes() {
+ return peerDatanodes;
+ }
+
+ public long getContainerID() {
+ return getId();
+ }
+
+ public static ReconcileContainerCommand getFromProtobuf(ReconcileContainerCommandProto protoMessage) {
+ Preconditions.checkNotNull(protoMessage);
+
+ List<HddsProtos.DatanodeDetailsProto> peers = protoMessage.getPeersList();
+ Set<DatanodeDetails> peerNodes = !peers.isEmpty()
+ ? peers.stream()
+ .map(DatanodeDetails::getFromProtoBuf)
+ .collect(Collectors.toSet())
+ : emptySet();
+
+ return new ReconcileContainerCommand(protoMessage.getContainerID(), peerNodes);
+ }
+
+ @Override
+ public String toString() {
+ return getType() +
+ ": containerId=" + getContainerID() +
+ ", peerNodes=" + peerDatanodes;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReconcileContainerCommand that = (ReconcileContainerCommand) o;
+ return getContainerID() == that.getContainerID() &&
+ Objects.equals(peerDatanodes, that.peerDatanodes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getContainerID(), peerDatanodes);
+ }
+}
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeTestUtils.java
index 811e4b483a,0000000000..050eb4646f
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeTestUtils.java
@@@ -1,358 -1,0 +1,359 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.checksum;
+
+import static org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager.getContainerChecksumFile;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
- import java.io.FileInputStream;
- import java.io.FileOutputStream;
+import java.io.IOException;
++import java.io.InputStream;
++import java.io.OutputStream;
+import java.nio.ByteBuffer;
++import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.ozone.HddsDatanodeService;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+
+/**
+ * Helper methods for testing container checksum tree files and container reconciliation.
+ */
+public final class ContainerMerkleTreeTestUtils {
+ private ContainerMerkleTreeTestUtils() {
+ }
+
+ public static void assertTreesSortedAndMatch(ContainerProtos.ContainerMerkleTree expectedTree,
+ ContainerProtos.ContainerMerkleTree actualTree) {
+ assertEquals(expectedTree.getDataChecksum(), actualTree.getDataChecksum());
+ assertEquals(expectedTree.getBlockMerkleTreeCount(), actualTree.getBlockMerkleTreeCount());
+
+ long prevBlockID = -1;
+ for (int blockIndex = 0; blockIndex < expectedTree.getBlockMerkleTreeCount(); blockIndex++) {
+ ContainerProtos.BlockMerkleTree expectedBlockTree = expectedTree.getBlockMerkleTree(blockIndex);
+ ContainerProtos.BlockMerkleTree actualBlockTree = actualTree.getBlockMerkleTree(blockIndex);
+
+ // Blocks should be sorted by block ID.
+ long currentBlockID = actualBlockTree.getBlockID();
+ assertTrue(prevBlockID < currentBlockID);
+ prevBlockID = currentBlockID;
+
+ assertEquals(expectedBlockTree.getBlockID(), actualBlockTree.getBlockID());
+ assertEquals(expectedBlockTree.getDataChecksum(), actualBlockTree.getDataChecksum());
+
+ long prevChunkOffset = -1;
+ for (int chunkIndex = 0; chunkIndex < expectedBlockTree.getChunkMerkleTreeCount(); chunkIndex++) {
+ ContainerProtos.ChunkMerkleTree expectedChunkTree = expectedBlockTree.getChunkMerkleTree(chunkIndex);
+ ContainerProtos.ChunkMerkleTree actualChunkTree = actualBlockTree.getChunkMerkleTree(chunkIndex);
+
+ // Chunks should be sorted by offset.
+ long currentChunkOffset = actualChunkTree.getOffset();
+ assertTrue(prevChunkOffset < currentChunkOffset);
+ prevChunkOffset = currentChunkOffset;
+
+ assertEquals(expectedChunkTree.getOffset(), actualChunkTree.getOffset());
+ assertEquals(expectedChunkTree.getLength(), actualChunkTree.getLength());
+ assertEquals(expectedChunkTree.getDataChecksum(), actualChunkTree.getDataChecksum());
+ }
+ }
+ }
+
+ /**
+ * Builds a ChunkInfo object using the provided information. No new checksums are calculated, so this can be used
+ * as either the leaves of pre-computed merkle trees that serve as expected values, or as building blocks to pass
+ * to ContainerMerkleTree to have it build the whole tree from this information.
+ *
+ * @param indexInBlock Which chunk number within a block this is. The chunk's offset is automatically calculated
+ * from this based on a fixed length.
+ * @param chunkChecksums The checksums within the chunk. Each is assumed to apply to a fixed value
+ * "bytesPerChecksum" amount of data and are assumed to be contiguous.
+ * @return The ChunkInfo proto object built from this information.
+ */
+ public static ContainerProtos.ChunkInfo buildChunk(ConfigurationSource config, int indexInBlock,
+ ByteBuffer... chunkChecksums) {
+ final long chunkSize = (long) config.getStorageSize(
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
+ final int bytesPerChecksum = config.getObject(OzoneClientConfig.class).getBytesPerChecksum();
+
+ // Each chunk checksum is added under the same ChecksumData object.
+ ContainerProtos.ChecksumData checksumData = ContainerProtos.ChecksumData.newBuilder()
+ .setType(ContainerProtos.ChecksumType.CRC32)
+ .setBytesPerChecksum(bytesPerChecksum)
+ .addAllChecksums(Arrays.stream(chunkChecksums)
+ .map(ByteString::copyFrom)
+ .collect(Collectors.toList()))
+ .build();
+
+ return ContainerProtos.ChunkInfo.newBuilder()
+ .setChecksumData(checksumData)
+ .setChunkName("chunk")
+ .setOffset(indexInBlock)
+ .setLen(chunkSize)
+ .build();
+ }
+
+ /**
+ * This reads the checksum file for a container from the disk without synchronization/coordination between readers
+ * and writers within a datanode.
+ */
+ public static ContainerProtos.ContainerChecksumInfo readChecksumFile(ContainerData data) throws IOException {
- try (FileInputStream inStream = new FileInputStream(getContainerChecksumFile(data))) {
++ try (InputStream inStream = Files.newInputStream(getContainerChecksumFile(data).toPath())) {
+ return ContainerProtos.ContainerChecksumInfo.parseFrom(inStream);
+ }
+ }
+
+ /**
+ * Builds a {@link ContainerMerkleTreeWriter} representing arbitrary data. This can be used to test that the same
+ * structure is preserved throughout serialization, deserialization, and API calls.
+ */
+ public static ContainerMerkleTreeWriter buildTestTree(ConfigurationSource conf) {
+ return buildTestTree(conf, 5);
+ }
+
+ public static ContainerMerkleTreeWriter buildTestTree(ConfigurationSource conf, int numBlocks) {
+ ContainerMerkleTreeWriter tree = new ContainerMerkleTreeWriter();
+ byte byteValue = 1;
+ for (int blockIndex = 1; blockIndex <= numBlocks; blockIndex++) {
+ List<ContainerProtos.ChunkInfo> chunks = new ArrayList<>();
+ for (int chunkIndex = 0; chunkIndex < 4; chunkIndex++) {
+ chunks.add(buildChunk(conf, chunkIndex, ByteBuffer.wrap(new byte[]{byteValue++, byteValue++, byteValue++})));
+ }
+ tree.addChunks(blockIndex, chunks);
+ }
+ return tree;
+ }
+
+ /**
+ * Returns a Pair of merkle tree and the expected container diff for that merkle tree.
+ */
+ public static Pair<ContainerProtos.ContainerMerkleTree, ContainerDiffReport>
+ buildTestTreeWithMismatches(ContainerMerkleTreeWriter originalTree, int numMissingBlocks, int numMissingChunks,
+ int numCorruptChunks) {
+
+ ContainerProtos.ContainerMerkleTree.Builder treeBuilder = originalTree.toProto().toBuilder();
+ ContainerDiffReport diff = new ContainerDiffReport();
+
+ introduceMissingBlocks(treeBuilder, numMissingBlocks, diff);
+ introduceMissingChunks(treeBuilder, numMissingChunks, diff);
+ introduceCorruptChunks(treeBuilder, numCorruptChunks, diff);
+ ContainerProtos.ContainerMerkleTree build = treeBuilder.build();
+ return Pair.of(build, diff);
+ }
+
+ /**
+ * Introduces missing blocks by removing random blocks from the tree.
+ */
+ private static void introduceMissingBlocks(ContainerProtos.ContainerMerkleTree.Builder treeBuilder,
+ int numMissingBlocks,
+ ContainerDiffReport diff) {
+ // Set to track unique blocks selected for mismatches
+ Set<Integer> selectedBlocks = new HashSet<>();
+ Random random = new Random();
+ for (int i = 0; i < numMissingBlocks; i++) {
+ int randomBlockIndex;
+ do {
+ randomBlockIndex = random.nextInt(treeBuilder.getBlockMerkleTreeCount());
+ } while (selectedBlocks.contains(randomBlockIndex));
+ selectedBlocks.add(randomBlockIndex);
+ ContainerProtos.BlockMerkleTree blockMerkleTree = treeBuilder.getBlockMerkleTree(randomBlockIndex);
+ diff.addMissingBlock(blockMerkleTree);
+ treeBuilder.removeBlockMerkleTree(randomBlockIndex);
+ treeBuilder.setDataChecksum(random.nextLong());
+ }
+ }
+
+ /**
+ * Introduces missing chunks by removing random chunks from selected blocks.
+ */
+ private static void introduceMissingChunks(ContainerProtos.ContainerMerkleTree.Builder treeBuilder,
+ int numMissingChunks,
+ ContainerDiffReport diff) {
+ // Set to track unique blocks selected for mismatches
+ Random random = new Random();
+ for (int i = 0; i < numMissingChunks; i++) {
+ int randomBlockIndex = random.nextInt(treeBuilder.getBlockMerkleTreeCount());
+
+ // Work on the chosen block to remove a random chunk
+ ContainerProtos.BlockMerkleTree.Builder blockBuilder = treeBuilder.getBlockMerkleTreeBuilder(randomBlockIndex);
+ if (blockBuilder.getChunkMerkleTreeCount() > 0) {
+ int randomChunkIndex = random.nextInt(blockBuilder.getChunkMerkleTreeCount());
+ ContainerProtos.ChunkMerkleTree chunkMerkleTree = blockBuilder.getChunkMerkleTree(randomChunkIndex);
+ diff.addMissingChunk(blockBuilder.getBlockID(), chunkMerkleTree);
+ blockBuilder.removeChunkMerkleTree(randomChunkIndex);
+ blockBuilder.setDataChecksum(random.nextLong());
+ treeBuilder.setDataChecksum(random.nextLong());
+ }
+ }
+ }
+
+ /**
+ * Introduces corrupt chunks by altering the checksum and setting them as unhealthy,
+ * ensuring each chunk in a block is only selected once for corruption.
+ */
+ private static void introduceCorruptChunks(ContainerProtos.ContainerMerkleTree.Builder treeBuilder,
+ int numCorruptChunks,
+ ContainerDiffReport diff) {
+ Map<Integer, Set<Integer>> corruptedChunksByBlock = new HashMap<>();
+ Random random = new Random();
+
+ for (int i = 0; i < numCorruptChunks; i++) {
+ // Select a random block
+ int randomBlockIndex = random.nextInt(treeBuilder.getBlockMerkleTreeCount());
+ ContainerProtos.BlockMerkleTree.Builder blockBuilder = treeBuilder.getBlockMerkleTreeBuilder(randomBlockIndex);
+
+ // Ensure each chunk in the block is only corrupted once
+ Set<Integer> corruptedChunks = corruptedChunksByBlock.computeIfAbsent(randomBlockIndex, k -> new HashSet<>());
+ if (corruptedChunks.size() < blockBuilder.getChunkMerkleTreeCount()) {
+ int randomChunkIndex;
+ do {
+ randomChunkIndex = random.nextInt(blockBuilder.getChunkMerkleTreeCount());
+ } while (corruptedChunks.contains(randomChunkIndex));
+ corruptedChunks.add(randomChunkIndex);
+
+ // Corrupt the selected chunk
+ ContainerProtos.ChunkMerkleTree.Builder chunkBuilder = blockBuilder.getChunkMerkleTreeBuilder(randomChunkIndex);
+ diff.addCorruptChunk(blockBuilder.getBlockID(), chunkBuilder.build());
+ chunkBuilder.setDataChecksum(chunkBuilder.getDataChecksum() + random.nextInt(1000) + 1);
+ chunkBuilder.setIsHealthy(false);
+ blockBuilder.setDataChecksum(random.nextLong());
+ treeBuilder.setDataChecksum(random.nextLong());
+ }
+ }
+ }
+
+ public static void assertContainerDiffMatch(ContainerDiffReport expectedDiff,
+ ContainerDiffReport actualDiff) {
+ assertNotNull(expectedDiff, "Expected diff is null");
+ assertNotNull(actualDiff, "Actual diff is null");
+ assertEquals(expectedDiff.getMissingBlocks().size(), actualDiff.getMissingBlocks().size(),
+ "Mismatch in number of missing blocks");
+ assertEquals(expectedDiff.getMissingChunks().size(), actualDiff.getMissingChunks().size(),
+ "Mismatch in number of missing chunks");
+ assertEquals(expectedDiff.getCorruptChunks().size(), actualDiff.getCorruptChunks().size(),
+ "Mismatch in number of corrupt chunks");
+
+ List<ContainerProtos.BlockMerkleTree> expectedMissingBlocks = expectedDiff.getMissingBlocks().stream().sorted(
+ Comparator.comparing(ContainerProtos.BlockMerkleTree::getBlockID)).collect(Collectors.toList());
+ List<ContainerProtos.BlockMerkleTree> actualMissingBlocks = expectedDiff.getMissingBlocks().stream().sorted(
+ Comparator.comparing(ContainerProtos.BlockMerkleTree::getBlockID)).collect(Collectors.toList());
+ for (int i = 0; i < expectedMissingBlocks.size(); i++) {
+ ContainerProtos.BlockMerkleTree expectedBlockMerkleTree = expectedMissingBlocks.get(i);
+ ContainerProtos.BlockMerkleTree actualBlockMerkleTree = actualMissingBlocks.get(i);
+ assertEquals(expectedBlockMerkleTree.getBlockID(), actualBlockMerkleTree.getBlockID());
+ assertEquals(expectedBlockMerkleTree.getChunkMerkleTreeCount(),
+ actualBlockMerkleTree.getChunkMerkleTreeCount());
+ assertEquals(expectedBlockMerkleTree.getDataChecksum(), actualBlockMerkleTree.getDataChecksum());
+ assertEqualsChunkMerkleTree(expectedBlockMerkleTree.getChunkMerkleTreeList(),
+ actualBlockMerkleTree.getChunkMerkleTreeList(), expectedBlockMerkleTree.getBlockID());
+ }
+
+ // Check missing chunks
+ Map<Long, List<ContainerProtos.ChunkMerkleTree>> expectedMissingChunks = expectedDiff.getMissingChunks();
+ Map<Long, List<ContainerProtos.ChunkMerkleTree>> actualMissingChunks = actualDiff.getMissingChunks();
+
+ for (Map.Entry<Long, List<ContainerProtos.ChunkMerkleTree>> entry : expectedMissingChunks.entrySet()) {
+ Long blockId = entry.getKey();
+ List<ContainerProtos.ChunkMerkleTree> expectedChunks = entry.getValue().stream().sorted(
+ Comparator.comparing(ContainerProtos.ChunkMerkleTree::getOffset)).collect(Collectors.toList());
+ List<ContainerProtos.ChunkMerkleTree> actualChunks = actualMissingChunks.get(blockId).stream().sorted(
+ Comparator.comparing(ContainerProtos.ChunkMerkleTree::getOffset)).collect(Collectors.toList());
+
+ assertNotNull(actualChunks, "Missing chunks for block " + blockId + " not found in actual diff");
+ assertEquals(expectedChunks.size(), actualChunks.size(),
+ "Mismatch in number of missing chunks for block " + blockId);
+ assertEqualsChunkMerkleTree(expectedChunks, actualChunks, blockId);
+ }
+
+ // Check corrupt chunks
+ Map<Long, List<ContainerProtos.ChunkMerkleTree>> expectedCorruptChunks = expectedDiff.getCorruptChunks();
+ Map<Long, List<ContainerProtos.ChunkMerkleTree>> actualCorruptChunks = actualDiff.getCorruptChunks();
+
+ for (Map.Entry<Long, List<ContainerProtos.ChunkMerkleTree>> entry : expectedCorruptChunks.entrySet()) {
+ Long blockId = entry.getKey();
+ List<ContainerProtos.ChunkMerkleTree> expectedChunks = entry.getValue().stream().sorted(
+ Comparator.comparing(ContainerProtos.ChunkMerkleTree::getOffset)).collect(Collectors.toList());
+ List<ContainerProtos.ChunkMerkleTree> actualChunks = actualCorruptChunks.get(blockId).stream().sorted(
+ Comparator.comparing(ContainerProtos.ChunkMerkleTree::getOffset)).collect(Collectors.toList());
+
+ assertNotNull(actualChunks, "Corrupt chunks for block " + blockId + " not found in actual diff");
+ assertEquals(expectedChunks.size(), actualChunks.size(),
+ "Mismatch in number of corrupt chunks for block " + blockId);
+ assertEqualsChunkMerkleTree(expectedChunks, actualChunks, blockId);
+ }
+ }
+
+ private static void assertEqualsChunkMerkleTree(List<ContainerProtos.ChunkMerkleTree> expectedChunkMerkleTreeList,
+ List<ContainerProtos.ChunkMerkleTree> actualChunkMerkleTreeList,
+ Long blockId) {
+ assertEquals(expectedChunkMerkleTreeList.size(), actualChunkMerkleTreeList.size());
+ for (int j = 0; j < expectedChunkMerkleTreeList.size(); j++) {
+ ContainerProtos.ChunkMerkleTree expectedChunk = expectedChunkMerkleTreeList.get(j);
+ ContainerProtos.ChunkMerkleTree actualChunk = actualChunkMerkleTreeList.get(j);
+ assertEquals(expectedChunk.getOffset(), actualChunk.getOffset(), "Mismatch in chunk offset for block "
+ + blockId);
+ assertEquals(expectedChunk.getDataChecksum(), actualChunk.getDataChecksum(),
+ "Mismatch in chunk checksum for block " + blockId);
+ }
+ }
+
+ /**
+ * This function checks whether the container checksum file exists.
+ */
+ public static boolean containerChecksumFileExists(HddsDatanodeService hddsDatanode,
+ ContainerInfo containerInfo) {
+ OzoneContainer ozoneContainer = hddsDatanode.getDatanodeStateMachine().getContainer();
+ Container container = ozoneContainer.getController().getContainer(containerInfo.getContainerID());
+ return ContainerChecksumTreeManager.checksumFileExist(container);
+ }
+
+ public static void writeContainerDataTreeProto(ContainerData data, ContainerProtos.ContainerMerkleTree tree)
+ throws IOException {
+ ContainerProtos.ContainerChecksumInfo checksumInfo = ContainerProtos.ContainerChecksumInfo.newBuilder()
+ .setContainerID(data.getContainerID())
+ .setContainerMerkleTree(tree).build();
+ File checksumFile = getContainerChecksumFile(data);
+
- try (FileOutputStream outputStream = new FileOutputStream(checksumFile)) {
++ try (OutputStream outputStream = Files.newOutputStream(checksumFile.toPath())) {
+ checksumInfo.writeTo(outputStream);
+ } catch (IOException ex) {
+ throw new IOException("Error occurred when writing container merkle tree for containerID "
+ + data.getContainerID(), ex);
+ }
+ data.setDataChecksum(checksumInfo.getContainerMerkleTree().getDataChecksum());
+ }
+}
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
index ab8929c18c,9ef89afd3c..626fabfb33
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java
@@@ -17,7 -17,7 +17,8 @@@
package org.apache.hadoop.ozone.container.common;
+ import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@@ -53,20 -47,15 +54,22 @@@ import org.apache.hadoop.io.retry.Retry
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
+ import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
+import org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeWriter;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+ import org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
@@@ -79,16 -68,11 +82,17 @@@ import org.apache.hadoop.ozone.containe
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
+ import org.apache.hadoop.ozone.container.common.volume.VolumeChoosingPolicyFactory;
+import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScanError;
+import org.apache.hadoop.ozone.container.ozoneimpl.DataScanResult;
+import org.apache.hadoop.ozone.container.ozoneimpl.MetadataScanResult;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
@@@ -428,26 -354,12 +432,35 @@@ public final class ContainerTestUtils
null, null);
}
+ /** Initialize {@link DatanodeLayoutStorage}. Normally this is done during {@link HddsDatanodeService} start,
+ * have to do the same for tests that create {@link OzoneContainer} manually. */
+ public static void initializeDatanodeLayout(ConfigurationSource conf, DatanodeDetails dn) throws IOException {
+ DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(conf, dn.getUuidString());
+ if (layoutStorage.getState() != INITIALIZED) {
+ layoutStorage.initialize();
+ }
+ }
++
+ /**
+ * Creates block metadata for the given container with the specified number of blocks and chunks per block.
+ */
+ public static void createBlockMetaData(KeyValueContainerData data, int numOfBlocksPerContainer,
+ int numOfChunksPerBlock) throws IOException {
+ try (DBHandle metadata = BlockUtils.getDB(data, new OzoneConfiguration())) {
+ for (int j = 0; j < numOfBlocksPerContainer; j++) {
+ BlockID blockID = new BlockID(data.getContainerID(), j);
+ String blockKey = data.getBlockKey(blockID.getLocalID());
+ BlockData kd = new BlockData(blockID);
+ List<ContainerProtos.ChunkInfo> chunks = Lists.newArrayList();
+ for (int k = 0; k < numOfChunksPerBlock; k++) {
+ long dataLen = 10L;
+ ChunkInfo chunkInfo = ContainerTestHelper.getChunk(blockID.getLocalID(), k, k * dataLen, dataLen);
+ ContainerTestHelper.setDataChecksum(chunkInfo, ContainerTestHelper.getData((int) dataLen));
+ chunks.add(chunkInfo.getProtoBufMessage());
+ }
+ kd.setChunks(chunks);
+ metadata.getStore().getBlockDataTable().put(blockKey, kd);
+ }
+ }
+ }
}
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 878e40e30e,d97651822d..f7e8b6d085
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@@ -57,9 -48,9 +55,10 @@@ import java.util.List
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.BlockID;
@@@ -1024,10 -1025,12 +1025,10 @@@ public class TestBlockDeletingService
dnConf.setBlockDeletionLimit(10);
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
conf.setFromObject(dnConf);
- ContainerSet containerSet = new ContainerSet(1000);
+ ContainerSet containerSet = newContainerSet();
ContainerMetrics metrics = ContainerMetrics.create(conf);
KeyValueHandler keyValueHandler =
- new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet,
- metrics, c -> {
- });
+ ContainerTestUtils.getKeyValueHandler(conf, datanodeUuid, containerSet, volumeSet, metrics);
int containerCount = 5;
int blocksPerContainer = 3;
createToDeleteBlocks(containerSet, containerCount,
@@@ -1080,47 -1083,6 +1081,47 @@@
}
}
+ /**
+ * The container checksum tree file is updated with the blocks that have been deleted after the on disk block files
+ * are removed from disk, but before the transaction is removed from the DB. If there is a failure partway through,
+ * the checksum tree file should still get updated when the transaction is retried, even if the block file is not
+ * present.
+ */
+ @ContainerTestVersionInfo.ContainerTest
+ public void testChecksumFileUpdatedWhenDeleteRetried(ContainerTestVersionInfo versionInfo) throws Exception {
+ final int numBlocks = 4;
+ setLayoutAndSchemaForTest(versionInfo);
+ DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class);
+ dnConf.setBlockDeletionLimit(4);
+ this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
+ conf.setFromObject(dnConf);
- ContainerSet containerSet = new ContainerSet(1000);
++ ContainerSet containerSet = newContainerSet();
+ KeyValueContainerData contData = createToDeleteBlocks(containerSet, numBlocks, 4);
+ KeyValueHandler keyValueHandler =
+ ContainerTestUtils.getKeyValueHandler(conf, datanodeUuid, containerSet, volumeSet);
+ BlockDeletingServiceTestImpl svc =
+ getBlockDeletingService(containerSet, conf, keyValueHandler);
+ svc.start();
+ GenericTestUtils.waitFor(svc::isStarted, 100, 3000);
+
+ // Remove all the block files from the disk, as if they were deleted previously but the system failed before
+ // doing any metadata updates or removing the transaction of to-delete block IDs from the DB.
+ File blockDataDir = new File(contData.getChunksPath());
+ try (DirectoryStream<Path> stream = Files.newDirectoryStream(blockDataDir.toPath())) {
+ for (Path entry : stream) {
+ assertTrue(entry.toFile().delete());
+ }
+ }
+
+ String[] blockFilesRemaining = blockDataDir.list();
+ assertNotNull(blockFilesRemaining);
+ assertEquals(0, blockFilesRemaining.length);
+
+ deleteAndWait(svc, 1);
+
+ assertDeletionsInChecksumFile(contData, numBlocks);
+ }
+
/**
* Check blockData record count of certain container (DBHandle not provided).
*
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java
index 236dfdb466,8730ea69f5..e8ed604829
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java
@@@ -128,10 -130,11 +129,10 @@@ public class TestSchemaTwoBackwardsComp
StorageVolume.VolumeType.DATA_VOLUME, null);
blockManager = new BlockManagerImpl(conf);
- chunkManager = new FilePerBlockStrategy(true, blockManager, volumeSet);
+ chunkManager = new FilePerBlockStrategy(true, blockManager);
- containerSet = new ContainerSet(1000);
+ containerSet = newContainerSet();
- keyValueHandler = new KeyValueHandler(conf, datanodeUuid,
- containerSet, volumeSet, ContainerMetrics.create(conf), c -> { });
+ keyValueHandler = ContainerTestUtils.getKeyValueHandler(conf, datanodeUuid, containerSet, volumeSet);
ozoneContainer = mock(OzoneContainer.class);
when(ozoneContainer.getContainerSet()).thenReturn(containerSet);
when(ozoneContainer.getWriteChannel()).thenReturn(null);
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index 12ed53f0f1,6afcadb809..bd72c358f6
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@@ -69,6 -72,6 +72,7 @@@ import org.apache.hadoop.ozone.common.C
import org.apache.hadoop.ozone.common.OzoneChecksumException;
import org.apache.hadoop.ozone.common.utils.BufferUtils;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
++import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
@@@ -132,7 -151,16 +152,16 @@@ public class TestHddsDispatcher
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(),
scmId.toString());
containerSet.addContainer(container);
- HddsDispatcher hddsDispatcher = ContainerTestUtils.getHddsDispatcher(conf, containerSet, volumeSet, context);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerType containerType : ContainerType.values()) {
- handlers.put(containerType,
- Handler.getHandlerForContainerType(containerType, conf,
- context.getParent().getDatanodeDetails().getUuidString(),
- containerSet, volumeSet, volumeChoosingPolicy, metrics, NO_OP_ICR_SENDER));
++ handlers.put(containerType, Handler.getHandlerForContainerType(containerType, conf,
++ context.getParent().getDatanodeDetails().getUuidString(),
++ containerSet, volumeSet, volumeChoosingPolicy, metrics, NO_OP_ICR_SENDER,
++ new ContainerChecksumTreeManager(conf)));
+ }
+ HddsDispatcher hddsDispatcher = new HddsDispatcher(
+ conf, containerSet, volumeSet, handlers, context, metrics, null);
hddsDispatcher.setClusterId(scmId.toString());
ContainerCommandResponseProto responseOne = hddsDispatcher
.dispatch(getWriteChunkRequest(dd.getUuidString(), 1L, 1L), null);
@@@ -259,10 -288,19 +289,20 @@@
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(),
scmId.toString());
containerSet.addContainer(container);
- HddsDispatcher hddsDispatcher = ContainerTestUtils.getHddsDispatcher(conf, containerSet, volumeSet, context);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerType containerType : ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(containerType, conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
- containerSet, volumeSet, volumeChoosingPolicy, metrics, NO_OP_ICR_SENDER));
++ containerSet, volumeSet, volumeChoosingPolicy, metrics, NO_OP_ICR_SENDER,
++ new ContainerChecksumTreeManager(conf)));
+ }
+ HddsDispatcher hddsDispatcher = new HddsDispatcher(
+ conf, containerSet, volumeSet, handlers, context, metrics, null);
hddsDispatcher.setClusterId(scmId.toString());
- containerData.getVolume().getVolumeInfo()
- .ifPresent(volumeInfo -> volumeInfo.incrementUsedSpace(50));
+ containerData.getVolume().getVolumeUsage()
+ .ifPresent(usage -> usage.incrementUsedSpace(50));
usedSpace.addAndGet(50);
ContainerCommandResponseProto response = hddsDispatcher
.dispatch(getWriteChunkRequest(dd.getUuidString(), 1L, 1L), null);
@@@ -499,8 -535,17 +537,18 @@@
}
});
StateContext context = ContainerTestUtils.getMockContext(dd, conf);
- final HddsDispatcher hddsDispatcher =
- ContainerTestUtils.getHddsDispatcher(conf, containerSet, volumeSet, context, tokenVerifier);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerType containerType : ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(containerType, conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
- containerSet, volumeSet, volumeChoosingPolicy, metrics, NO_OP_ICR_SENDER));
++ containerSet, volumeSet, volumeChoosingPolicy, metrics, NO_OP_ICR_SENDER,
++ new ContainerChecksumTreeManager(conf)));
+ }
+
+ final HddsDispatcher hddsDispatcher = new HddsDispatcher(conf,
+ containerSet, volumeSet, handlers, context, metrics, tokenVerifier);
hddsDispatcher.setClusterId(scmId.toString());
return hddsDispatcher;
}
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
index 99a86af65a,829f8c55ee..b2d688400c
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
@@@ -24,6 -26,6 +26,7 @@@ import java.util.Map
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
++import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@@@ -56,7 -61,19 +62,20 @@@ public class TestHandler
DatanodeDetails datanodeDetails = mock(DatanodeDetails.class);
StateContext context = ContainerTestUtils.getMockContext(
datanodeDetails, conf);
- this.dispatcher = ContainerTestUtils.getHddsDispatcher(conf, containerSet, volumeSet, context);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerProtos.ContainerType containerType :
+ ContainerProtos.ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(
+ containerType, conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
+ containerSet, volumeSet, volumeChoosingPolicy, metrics,
- TestHddsDispatcher.NO_OP_ICR_SENDER));
++ TestHddsDispatcher.NO_OP_ICR_SENDER,
++ new ContainerChecksumTreeManager(conf)));
+ }
+ this.dispatcher = new HddsDispatcher(
+ conf, containerSet, volumeSet, handlers, null, metrics, null);
}
@AfterEach
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReconcileContainerCommandHandler.java
index d7d01f9349,0000000000..80f2747d4c
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReconcileContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReconcileContainerCommandHandler.java
@@@ -1,187 -1,0 +1,188 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import static java.util.Collections.singletonMap;
+import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
+import static org.apache.hadoop.ozone.OzoneConsts.GB;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createBlockMetaData;
++import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
+import org.apache.hadoop.ozone.container.checksum.DNContainerOperationClient;
+import org.apache.hadoop.ozone.container.checksum.ReconcileContainerTask;
+import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
+import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+import org.apache.hadoop.ozone.container.common.report.IncrementalReportSender;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
+import org.apache.hadoop.ozone.protocol.commands.ReconcileContainerCommand;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tests Datanode handling of reconcile container commands.
+ */
+public class TestReconcileContainerCommandHandler {
+ public static final Logger LOG = LoggerFactory.getLogger(TestReconcileContainerCommandHandler.class);
+
+ private static final int NUM_CONTAINERS = 3;
+
+ private ContainerSet containerSet;
+ private OzoneContainer ozoneContainer;
+ private StateContext context;
+ private ReconcileContainerCommandHandler subject;
+ private ReplicationSupervisor mockSupervisor;
+ @TempDir
+ private Path tempDir;
+ @TempDir
+ private Path dbFile;
+
+ public void init(ContainerLayoutVersion layout, IncrementalReportSender<Container> icrSender)
+ throws Exception {
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ DatanodeDetails dnDetails = randomDatanodeDetails();
+
+ mockSupervisor = mock(ReplicationSupervisor.class);
+ doAnswer(invocation -> {
+ ((ReconcileContainerTask)invocation.getArguments()[0]).runTask();
+ return null;
+ }).when(mockSupervisor).addTask(any());
+
+ subject = new ReconcileContainerCommandHandler(mockSupervisor, mock(DNContainerOperationClient.class));
+ context = ContainerTestUtils.getMockContext(dnDetails, conf);
+
- containerSet = new ContainerSet(1000);
++ containerSet = newContainerSet();
+ for (int id = 1; id <= NUM_CONTAINERS; id++) {
+ KeyValueContainerData data = new KeyValueContainerData(id, layout, GB,
+ PipelineID.randomId().toString(), randomDatanodeDetails().getUuidString());
+ data.setMetadataPath(tempDir.toString());
+ data.setDbFile(dbFile.toFile());
+ createBlockMetaData(data, 5, 3);
+ containerSet.addContainer(new KeyValueContainer(data, conf));
+ }
+
+ assertEquals(NUM_CONTAINERS, containerSet.containerCount());
+
+ Handler containerHandler = new KeyValueHandler(new OzoneConfiguration(), dnDetails.getUuidString(), containerSet,
+ mock(VolumeSet.class), mock(ContainerMetrics.class), icrSender, new ContainerChecksumTreeManager(conf));
+ ContainerController controller = new ContainerController(containerSet,
+ singletonMap(ContainerProtos.ContainerType.KeyValueContainer, containerHandler));
+ ozoneContainer = mock(OzoneContainer.class);
+ when(ozoneContainer.getController()).thenReturn(controller);
+ when(ozoneContainer.getContainerSet()).thenReturn(containerSet);
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void testReconcileContainerCommandReports(ContainerLayoutVersion layout) throws Exception {
+ Map<ContainerID, ContainerReplicaProto> containerReportsSent = new HashMap<>();
+ IncrementalReportSender<Container> icrSender = c -> {
+ try {
+ ContainerID id = ContainerID.valueOf(c.getContainerData().getContainerID());
+ containerReportsSent.put(id, c.getContainerReport());
+ LOG.info("Added container report for container {}", id);
+ } catch (Exception ex) {
+ LOG.error("ICR sender failed", ex);
+ }
+ };
+ init(layout, icrSender);
+
+ for (int id = 1; id <= NUM_CONTAINERS; id++) {
+ ReconcileContainerCommand cmd = new ReconcileContainerCommand(id, Collections.emptySet());
+ subject.handle(cmd, ozoneContainer, context, null);
+ }
+
+ // An unknown container should not trigger a container report being sent.
+ ReconcileContainerCommand unknownContainerCmd = new ReconcileContainerCommand(NUM_CONTAINERS + 1,
+ Collections.emptySet());
+ subject.handle(unknownContainerCmd, ozoneContainer, context, null);
+
+ // Since the replication supervisor is mocked in this test, reports are processed immediately.
+ verifyAllContainerReports(containerReportsSent);
+ }
+
+ /**
+ * Most metrics are handled by the ReplicationSupervisor. Only check the individual metrics here.
+ */
+ @ContainerLayoutTestInfo.ContainerTest
+ public void testReconcileContainerCommandMetrics(ContainerLayoutVersion layout) throws Exception {
+ init(layout, c -> { });
+
+ assertEquals(0, subject.getInvocationCount());
+
+ for (int id = 1; id <= NUM_CONTAINERS; id++) {
+ ReconcileContainerCommand cmd = new ReconcileContainerCommand(id, Collections.emptySet());
+ subject.handle(cmd, ozoneContainer, context, null);
+ }
+
+ when(mockSupervisor.getReplicationRequestCount(subject.getMetricsName())).thenReturn(3L);
+ when(mockSupervisor.getReplicationRequestTotalTime(subject.getMetricsName())).thenReturn(10L);
+ when(mockSupervisor.getReplicationRequestAvgTime(subject.getMetricsName())).thenReturn(3L);
+ when(mockSupervisor.getReplicationQueuedCount(subject.getMetricsName())).thenReturn(1L);
+
+ assertEquals(subject.getMetricsName(), "ContainerReconciliations");
+ assertEquals(NUM_CONTAINERS, subject.getInvocationCount());
+ assertEquals(subject.getQueuedCount(), 1);
+ assertEquals(subject.getTotalRunTime(), 10);
+ assertEquals(subject.getAverageRunTime(), 3);
+ }
+
+ private void verifyAllContainerReports(Map<ContainerID, ContainerReplicaProto> reportsSent) {
+ assertEquals(NUM_CONTAINERS, reportsSent.size());
+
+ for (Map.Entry<ContainerID, ContainerReplicaProto> entry: reportsSent.entrySet()) {
+ ContainerID id = entry.getKey();
+ assertNotNull(containerSet.getContainer(id.getId()));
+
+ long sentDataChecksum = entry.getValue().getDataChecksum();
+ // Current implementation is incomplete, and uses a mocked checksum.
+ assertNotEquals(0, sentDataChecksum, "Report of container " + id +
+ " should have a non-zero checksum");
+ }
+ }
+}
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 33f4faefb6,76e696e91d..918511e1f6
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@@ -17,34 -17,16 +17,33 @@@
package org.apache.hadoop.ozone.container.keyvalue;
+import static java.nio.charset.StandardCharsets.UTF_8;
- import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.QUASI_CLOSED;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.GB;
+import static org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager.getContainerChecksumFile;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.writeContainerDataTreeProto;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.WRITE_STAGE;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createBlockMetaData;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createDbInstancesForTestIfNeeded;
+ import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
+import static org.apache.hadoop.ozone.container.keyvalue.TestContainerCorruptions.getBlock;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
- import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.atMostOnce;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
@@@ -116,14 -66,10 +115,15 @@@ import org.apache.hadoop.ozone.containe
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.util.Sets;
- import org.apache.ozone.test.GenericTestUtils;
+ import org.apache.hadoop.util.Time;
+ import org.apache.ozone.test.GenericTestUtils.LogCapturer;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
@@@ -361,34 -273,20 +360,19 @@@ public class TestKeyValueHandler
volumeSet = new MutableVolumeSet(UUID.randomUUID().toString(), conf,
null, StorageVolume.VolumeType.DATA_VOLUME, null);
try {
- ContainerSet cset = new ContainerSet(1000);
+ ContainerSet cset = newContainerSet();
+ int[] interval = new int[1];
+ interval[0] = 2;
- ContainerMetrics metrics = new ContainerMetrics(interval);
DatanodeDetails datanodeDetails = mock(DatanodeDetails.class);
StateContext context = ContainerTestUtils.getMockContext(
datanodeDetails, conf);
// Ensures that KeyValueHandler falls back to FILE_PER_BLOCK.
conf.set(OZONE_SCM_CONTAINER_LAYOUT_KEY, "FILE_PER_CHUNK");
- new KeyValueHandler(conf, context.getParent().getDatanodeDetails().getUuidString(), cset, volumeSet,
- metrics, c -> { });
+ ContainerTestUtils.getKeyValueHandler(conf,
+ context.getParent().getDatanodeDetails().getUuidString(), cset, volumeSet);
assertEquals(ContainerLayoutVersion.FILE_PER_BLOCK,
conf.getEnum(OZONE_SCM_CONTAINER_LAYOUT_KEY, ContainerLayoutVersion.FILE_PER_CHUNK));
-
- //Set a class which is not of sub class of VolumeChoosingPolicy
- conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY,
- "org.apache.hadoop.ozone.container.common.impl.HddsDispatcher");
- RuntimeException exception = assertThrows(RuntimeException.class,
- () -> ContainerTestUtils.getKeyValueHandler(conf, context.getParent().getDatanodeDetails().getUuidString(),
- cset, volumeSet));
-
- assertThat(exception).hasMessageEndingWith(
- "class org.apache.hadoop.ozone.container.common.impl.HddsDispatcher " +
- "not org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy");
} finally {
volumeSet.shutdown();
FileUtil.fullyDelete(datanodeDir);
@@@ -453,8 -346,8 +437,8 @@@
final long containerID = 1L;
final String clusterId = UUID.randomUUID().toString();
final String datanodeId = UUID.randomUUID().toString();
- final ConfigurationSource conf = new OzoneConfiguration();
+ conf = new OzoneConfiguration();
- final ContainerSet containerSet = new ContainerSet(1000);
+ final ContainerSet containerSet = newContainerSet();
final MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
HddsVolume hddsVolume = new HddsVolume.Builder(testDir).conf(conf)
@@@ -540,212 -432,17 +523,213 @@@
}
}
+ @ContainerLayoutTestInfo.ContainerTest
+ public void testContainerChecksumInvocation(ContainerLayoutVersion layoutVersion) throws Exception {
+ conf = new OzoneConfiguration();
+
+ KeyValueContainerData data = new KeyValueContainerData(123L, layoutVersion, GB,
+ PipelineID.randomId().toString(), randomDatanodeDetails().getUuidString());
+ data.setMetadataPath(tempDir.toString());
+ data.setDbFile(dbFile.toFile());
+
+ Container container = new KeyValueContainer(data, conf);
+ createBlockMetaData(data, 5, 3);
- ContainerSet containerSet = new ContainerSet(1000);
++ ContainerSet containerSet = newContainerSet();
+ containerSet.addContainer(container);
+
+ // Allows checking the invocation count of the lambda.
+ AtomicInteger icrCount = new AtomicInteger(0);
+ IncrementalReportSender<Container> icrSender = c -> {
+ // Check that the ICR contains expected info about the container.
+ ContainerReplicaProto report = c.getContainerReport();
+ long reportedID = report.getContainerID();
+ Assertions.assertEquals(container.getContainerData().getContainerID(), reportedID);
+
+ long reportDataChecksum = report.getDataChecksum();
+ assertNotEquals(0, reportDataChecksum,
+ "Container report should have populated the checksum field with a non-zero value.");
+ icrCount.incrementAndGet();
+ };
+
+ KeyValueHandler keyValueHandler = new KeyValueHandler(conf, randomDatanodeDetails().getUuidString(), containerSet,
+ mock(MutableVolumeSet.class), mock(ContainerMetrics.class), icrSender, new ContainerChecksumTreeManager(conf));
+
+ Assertions.assertEquals(0, icrCount.get());
+ // This should trigger container report validation in the ICR handler above.
+ DNContainerOperationClient mockDnClient = mock(DNContainerOperationClient.class);
+ DatanodeDetails peer1 = MockDatanodeDetails.randomDatanodeDetails();
+ DatanodeDetails peer2 = MockDatanodeDetails.randomDatanodeDetails();
+ DatanodeDetails peer3 = MockDatanodeDetails.randomDatanodeDetails();
+ when(mockDnClient.getContainerChecksumInfo(anyLong(), any())).thenReturn(null);
+ keyValueHandler.reconcileContainer(mockDnClient, container, Sets.newHashSet(peer1, peer2, peer3));
+ // Make sure all the replicas are used for reconciliation.
+ Mockito.verify(mockDnClient, atMostOnce()).getContainerChecksumInfo(anyLong(), eq(peer1));
+ Mockito.verify(mockDnClient, atMostOnce()).getContainerChecksumInfo(anyLong(), eq(peer2));
+ Mockito.verify(mockDnClient, atMostOnce()).getContainerChecksumInfo(anyLong(), eq(peer3));
+ Assertions.assertEquals(1, icrCount.get());
+ }
+
+ @ParameterizedTest
+ @MethodSource("corruptionValues")
+ public void testFullContainerReconciliation(int numBlocks, int numChunks) throws Exception {
+ KeyValueHandler kvHandler = createKeyValueHandler(tempDir);
+ ContainerChecksumTreeManager checksumManager = kvHandler.getChecksumManager();
+ DNContainerOperationClient dnClient = new DNContainerOperationClient(conf, null, null);
+ final long containerID = 100L;
+ // Create 3 containers with 15 blocks each and 3 replicas.
+ List<KeyValueContainer> containers = createContainerWithBlocks(kvHandler, containerID, 15, 3);
+ assertEquals(3, containers.size());
+
+ // Introduce corruption in each container on different replicas.
+ introduceCorruption(kvHandler, containers.get(1), numBlocks, numChunks, false);
+ introduceCorruption(kvHandler, containers.get(2), numBlocks, numChunks, true);
+
+ // Without reconciliation, checksums should be different because of the corruption.
+ Set<Long> checksumsBeforeReconciliation = new HashSet<>();
+ for (KeyValueContainer kvContainer : containers) {
+ Optional<ContainerProtos.ContainerChecksumInfo> containerChecksumInfo =
+ checksumManager.read(kvContainer.getContainerData());
+ assertTrue(containerChecksumInfo.isPresent());
+ long dataChecksum = containerChecksumInfo.get().getContainerMerkleTree().getDataChecksum();
+ assertEquals(kvContainer.getContainerData().getDataChecksum(), dataChecksum);
+ checksumsBeforeReconciliation.add(dataChecksum);
+ }
+ // There should be more than 1 checksum because of the corruption.
+ assertTrue(checksumsBeforeReconciliation.size() > 1);
+
+ List<DatanodeDetails> datanodes = ImmutableList.of(randomDatanodeDetails(), randomDatanodeDetails(),
+ randomDatanodeDetails());
+ Map<String, KeyValueContainer> dnToContainerMap = new HashMap<>();
+ dnToContainerMap.put(datanodes.get(0).getUuidString(), containers.get(0));
+ dnToContainerMap.put(datanodes.get(1).getUuidString(), containers.get(1));
+ dnToContainerMap.put(datanodes.get(2).getUuidString(), containers.get(2));
+
+ // Setup mock for each datanode network calls needed for reconciliation.
+ try (MockedStatic<ContainerProtocolCalls> containerProtocolMock =
+ Mockito.mockStatic(ContainerProtocolCalls.class)) {
+ mockContainerProtocolCalls(containerProtocolMock, dnToContainerMap, checksumManager, kvHandler, containerID);
+
+ kvHandler.reconcileContainer(dnClient, containers.get(0), Sets.newHashSet(datanodes));
+ kvHandler.reconcileContainer(dnClient, containers.get(1), Sets.newHashSet(datanodes));
+ kvHandler.reconcileContainer(dnClient, containers.get(2), Sets.newHashSet(datanodes));
+
+ // After reconciliation, checksums should be the same for all containers.
+ ContainerProtos.ContainerChecksumInfo prevContainerChecksumInfo = null;
+ for (KeyValueContainer kvContainer : containers) {
+ kvHandler.createContainerMerkleTree(kvContainer);
+ Optional<ContainerProtos.ContainerChecksumInfo> containerChecksumInfo =
+ checksumManager.read(kvContainer.getContainerData());
+ assertTrue(containerChecksumInfo.isPresent());
+ long dataChecksum = containerChecksumInfo.get().getContainerMerkleTree().getDataChecksum();
+ assertEquals(kvContainer.getContainerData().getDataChecksum(), dataChecksum);
+ if (prevContainerChecksumInfo != null) {
+ assertEquals(prevContainerChecksumInfo.getContainerMerkleTree().getDataChecksum(), dataChecksum);
+ }
+ prevContainerChecksumInfo = containerChecksumInfo.get();
+ }
+ }
+ }
++
+ private void mockContainerProtocolCalls(MockedStatic<ContainerProtocolCalls> containerProtocolMock,
+ Map<String, KeyValueContainer> dnToContainerMap,
+ ContainerChecksumTreeManager checksumManager,
+ KeyValueHandler kvHandler,
+ long containerID) {
+ // Mock getContainerChecksumInfo
+ containerProtocolMock.when(() -> ContainerProtocolCalls.getContainerChecksumInfo(any(), anyLong(), any()))
+ .thenAnswer(inv -> {
+ XceiverClientSpi xceiverClientSpi = inv.getArgument(0);
+ Pipeline pipeline = xceiverClientSpi.getPipeline();
+ assertEquals(1, pipeline.size());
+ DatanodeDetails dn = pipeline.getFirstNode();
+ KeyValueContainer container = dnToContainerMap.get(dn.getUuidString());
+ ByteString checksumInfo = checksumManager.getContainerChecksumInfo(container.getContainerData());
+ return ContainerProtos.GetContainerChecksumInfoResponseProto.newBuilder()
+ .setContainerID(containerID)
+ .setContainerChecksumInfo(checksumInfo)
+ .build();
+ });
+
+ // Mock getBlock
+ containerProtocolMock.when(() -> ContainerProtocolCalls.getBlock(any(), any(), any(), any(), anyMap()))
+ .thenAnswer(inv -> {
+ XceiverClientSpi xceiverClientSpi = inv.getArgument(0);
+ Pipeline pipeline = xceiverClientSpi.getPipeline();
+ assertEquals(1, pipeline.size());
+ DatanodeDetails dn = pipeline.getFirstNode();
+ KeyValueContainer container = dnToContainerMap.get(dn.getUuidString());
+ ContainerProtos.BlockData blockData = kvHandler.getBlockManager().getBlock(container, inv.getArgument(2))
+ .getProtoBufMessage();
+ return ContainerProtos.GetBlockResponseProto.newBuilder()
+ .setBlockData(blockData)
+ .build();
+ });
+
+ // Mock readChunk
+ containerProtocolMock.when(() -> ContainerProtocolCalls.readChunk(any(), any(), any(), any(), any()))
+ .thenAnswer(inv -> {
+ XceiverClientSpi xceiverClientSpi = inv.getArgument(0);
+ Pipeline pipeline = xceiverClientSpi.getPipeline();
+ assertEquals(1, pipeline.size());
+ DatanodeDetails dn = pipeline.getFirstNode();
+ KeyValueContainer container = dnToContainerMap.get(dn.getUuidString());
+ return createReadChunkResponse(inv, container, kvHandler);
+ });
+ }
+
+ // Helper method to create readChunk responses
+ private ContainerProtos.ReadChunkResponseProto createReadChunkResponse(InvocationOnMock inv,
+ KeyValueContainer container,
+ KeyValueHandler kvHandler) throws IOException {
+ ContainerProtos.DatanodeBlockID blockId = inv.getArgument(2);
+ ContainerProtos.ChunkInfo chunkInfo = inv.getArgument(1);
+ return ContainerProtos.ReadChunkResponseProto.newBuilder()
+ .setBlockID(blockId)
+ .setChunkData(chunkInfo)
+ .setData(kvHandler.getChunkManager().readChunk(container, BlockID.getFromProtobuf(blockId),
+ ChunkInfo.getFromProtoBuf(chunkInfo), null).toByteString())
+ .build();
+ }
+
+ @Test
+ public void testGetContainerChecksumInfoOnInvalidContainerStates() {
+ when(handler.handleGetContainerChecksumInfo(any(), any())).thenCallRealMethod();
+
+ // Only mock what is necessary for the request to fail. This test does not cover allowed states.
+ KeyValueContainer container = mock(KeyValueContainer.class);
+ KeyValueContainerData containerData = mock(KeyValueContainerData.class);
+ when(container.getContainerData()).thenReturn(containerData);
+
+ ContainerCommandRequestProto request = mock(ContainerCommandRequestProto.class);
+ when(request.hasGetContainerChecksumInfo()).thenReturn(true);
+ when(request.getCmdType()).thenReturn(ContainerProtos.Type.GetContainerChecksumInfo);
+ when(request.getTraceID()).thenReturn("123");
+
+ Set<State> disallowedStates = EnumSet.allOf(State.class);
+ disallowedStates.removeAll(EnumSet.of(CLOSED, QUASI_CLOSED, UNHEALTHY));
+
+ for (State state : disallowedStates) {
+ when(containerData.getState()).thenReturn(state);
+ ContainerProtos.ContainerCommandResponseProto response = handler.handleGetContainerChecksumInfo(request,
+ container);
+ assertNotNull(response);
+ assertEquals(ContainerProtos.Result.UNCLOSED_CONTAINER_IO, response.getResult());
+ assertTrue(response.getMessage().contains(state.toString()), "Response message did not contain the container " +
+ "state " + state);
+ }
+ }
+
@Test
public void testDeleteContainerTimeout() throws IOException {
final String testDir = tempDir.toString();
final long containerID = 1L;
final String clusterId = UUID.randomUUID().toString();
final String datanodeId = UUID.randomUUID().toString();
- final ConfigurationSource conf = new OzoneConfiguration();
+ conf = new OzoneConfiguration();
- final ContainerSet containerSet = new ContainerSet(1000);
+ final ContainerSet containerSet = newContainerSet();
final MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
final Clock clock = mock(Clock.class);
- long startTime = System.currentTimeMillis();
+ long startTime = Time.monotonicNow();
DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class);
when(clock.millis())
@@@ -773,8 -470,8 +757,8 @@@
final AtomicInteger icrReceived = new AtomicInteger(0);
final KeyValueHandler kvHandler = new KeyValueHandler(conf,
- datanodeId, containerSet, volumeSet, metrics,
- c -> icrReceived.incrementAndGet(), new ContainerChecksumTreeManager(conf), clock);
+ datanodeId, containerSet, volumeSet, null, metrics,
- c -> icrReceived.incrementAndGet(), clock);
++ c -> icrReceived.incrementAndGet(), clock, new ContainerChecksumTreeManager(conf));
kvHandler.setClusterID(clusterId);
final ContainerCommandRequestProto createContainer =
@@@ -808,185 -505,4 +792,185 @@@
.setContainerID(containerID).setPipelineID(UUID.randomUUID().toString())
.build();
}
+
+ private KeyValueHandler createKeyValueHandler(Path path) throws IOException {
- final ContainerSet containerSet = new ContainerSet(1000);
++ final ContainerSet containerSet = newContainerSet();
+ final MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
+
+ HddsVolume hddsVolume = new HddsVolume.Builder(path.toString()).conf(conf)
+ .clusterID(CLUSTER_ID).datanodeUuid(DATANODE_UUID)
+ .volumeSet(volumeSet)
+ .build();
+ hddsVolume.format(CLUSTER_ID);
+ hddsVolume.createWorkingDir(CLUSTER_ID, null);
+ hddsVolume.createTmpDirs(CLUSTER_ID);
+ when(volumeSet.getVolumesList()).thenReturn(Collections.singletonList(hddsVolume));
+ final KeyValueHandler kvHandler = ContainerTestUtils.getKeyValueHandler(conf,
+ DATANODE_UUID, containerSet, volumeSet);
+ kvHandler.setClusterID(CLUSTER_ID);
+ // Clean up metrics for next tests.
+ hddsVolume.getVolumeInfoStats().unregister();
+ hddsVolume.getVolumeIOStats().unregister();
+ ContainerMetrics.remove();
+ return kvHandler;
+ }
+
+ /**
+ * Creates a container with normal and deleted blocks.
+ * First it will insert normal blocks, and then it will insert
+ * deleted blocks.
+ */
+ protected List<KeyValueContainer> createContainerWithBlocks(KeyValueHandler kvHandler, long containerId,
+ int blocks, int numContainerCopy)
+ throws Exception {
+ String strBlock = "block";
+ String strChunk = "chunkFile";
+ List<KeyValueContainer> containers = new ArrayList<>();
+ MutableVolumeSet volumeSet = new MutableVolumeSet(DATANODE_UUID, conf, null,
+ StorageVolume.VolumeType.DATA_VOLUME, null);
+ createDbInstancesForTestIfNeeded(volumeSet, CLUSTER_ID, CLUSTER_ID, conf);
+ int bytesPerChecksum = 2 * (int) OzoneConsts.KB;
+ Checksum checksum = new Checksum(ContainerProtos.ChecksumType.SHA256,
+ bytesPerChecksum);
+ byte[] chunkData = RandomStringUtils.randomAscii(CHUNK_LEN).getBytes(UTF_8);
+ ChecksumData checksumData = checksum.computeChecksum(chunkData);
+
+ for (int j = 0; j < numContainerCopy; j++) {
+ KeyValueContainerData containerData = new KeyValueContainerData(containerId,
+ ContainerLayoutVersion.FILE_PER_BLOCK, (long) CHUNKS_PER_BLOCK * CHUNK_LEN * blocks,
+ UUID.randomUUID().toString(), UUID.randomUUID().toString());
+ Path kvContainerPath = Files.createDirectory(tempDir.resolve(containerId + "-" + j));
+ containerData.setMetadataPath(kvContainerPath.toString());
+ containerData.setDbFile(kvContainerPath.toFile());
+
+ KeyValueContainer container = new KeyValueContainer(containerData, conf);
+ StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList())
+ .forEach(hddsVolume -> hddsVolume.setDbParentDir(kvContainerPath.toFile()));
+ container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), UUID.randomUUID().toString());
+ assertNotNull(containerData.getChunksPath());
+ File chunksPath = new File(containerData.getChunksPath());
+ ContainerLayoutTestInfo.FILE_PER_BLOCK.validateFileCount(chunksPath, 0, 0);
+
+ List<ContainerProtos.ChunkInfo> chunkList = new ArrayList<>();
+ for (int i = 0; i < blocks; i++) {
+ BlockID blockID = new BlockID(containerId, i);
+ BlockData blockData = new BlockData(blockID);
+
+ chunkList.clear();
+ for (long chunkCount = 0; chunkCount < CHUNKS_PER_BLOCK; chunkCount++) {
+ String chunkName = strBlock + i + strChunk + chunkCount;
+ long offset = chunkCount * CHUNK_LEN;
+ ChunkInfo info = new ChunkInfo(chunkName, offset, CHUNK_LEN);
+ info.setChecksumData(checksumData);
+ chunkList.add(info.getProtoBufMessage());
+ kvHandler.getChunkManager().writeChunk(container, blockID, info,
+ ByteBuffer.wrap(chunkData), WRITE_STAGE);
+ }
+ kvHandler.getChunkManager().finishWriteChunks(container, blockData);
+ blockData.setChunks(chunkList);
+ blockData.setBlockCommitSequenceId(i);
+ kvHandler.getBlockManager().putBlock(container, blockData);
+ }
+
+ ContainerLayoutTestInfo.FILE_PER_BLOCK.validateFileCount(chunksPath, blocks, (long) blocks * CHUNKS_PER_BLOCK);
+ container.markContainerForClose();
+ kvHandler.closeContainer(container);
+ containers.add(container);
+ }
+
+ return containers;
+ }
+
+ /**
+ * Introduce corruption in the container.
+ * 1. Delete blocks from the container.
+ * 2. Corrupt chunks at an offset.
+ * If revers is true, the blocks and chunks are deleted in reverse order.
+ */
+ private void introduceCorruption(KeyValueHandler kvHandler, KeyValueContainer keyValueContainer, int numBlocks,
+ int numChunks, boolean reverse) throws IOException {
+ Random random = new Random();
+ KeyValueContainerData containerData = keyValueContainer.getContainerData();
+ // Simulate missing blocks
+ try (DBHandle handle = BlockUtils.getDB(containerData, conf);
+ BatchOperation batch = handle.getStore().getBatchHandler().initBatchOperation()) {
+ List<BlockData> blockDataList = kvHandler.getBlockManager().listBlock(keyValueContainer, -1, 100);
+ int size = blockDataList.size();
+ for (int i = 0; i < numBlocks; i++) {
+ BlockData blockData = reverse ? blockDataList.get(size - 1 - i) : blockDataList.get(i);
+ File blockFile = getBlock(keyValueContainer, blockData.getBlockID().getLocalID());
+ Assertions.assertTrue(blockFile.delete());
+ handle.getStore().getBlockDataTable().deleteWithBatch(batch, containerData.getBlockKey(blockData.getLocalID()));
+ }
+ handle.getStore().getBatchHandler().commitBatchOperation(batch);
+ }
+ Files.deleteIfExists(getContainerChecksumFile(keyValueContainer.getContainerData()).toPath());
+ kvHandler.createContainerMerkleTree(keyValueContainer);
+
+ // Corrupt chunks at an offset.
+ List<BlockData> blockDataList = kvHandler.getBlockManager().listBlock(keyValueContainer, -1, 100);
+ int size = blockDataList.size();
+ for (int i = 0; i < numChunks; i++) {
+ int blockIndex = reverse ? size - 1 - (i % size) : i % size;
+ BlockData blockData = blockDataList.get(blockIndex);
+ int chunkIndex = i / size;
+ File blockFile = getBlock(keyValueContainer, blockData.getBlockID().getLocalID());
+ List<ContainerProtos.ChunkInfo> chunks = new ArrayList<>(blockData.getChunks());
+ ContainerProtos.ChunkInfo chunkInfo = chunks.remove(chunkIndex);
+ corruptFileAtOffset(blockFile, (int) chunkInfo.getOffset(), (int) chunkInfo.getLen());
+
+ // TODO: On-demand scanner (HDDS-10374) should detect this corruption and generate container merkle tree.
+ ContainerProtos.ContainerChecksumInfo.Builder builder = kvHandler.getChecksumManager()
+ .read(containerData).get().toBuilder();
+ List<ContainerProtos.BlockMerkleTree> blockMerkleTreeList = builder.getContainerMerkleTree()
+ .getBlockMerkleTreeList();
+ assertEquals(size, blockMerkleTreeList.size());
+
+ builder.getContainerMerkleTreeBuilder().clearBlockMerkleTree();
+ for (int j = 0; j < blockMerkleTreeList.size(); j++) {
+ ContainerProtos.BlockMerkleTree.Builder blockMerkleTreeBuilder = blockMerkleTreeList.get(j).toBuilder();
+ if (j == blockIndex) {
+ List<ContainerProtos.ChunkMerkleTree.Builder> chunkMerkleTreeBuilderList =
+ blockMerkleTreeBuilder.getChunkMerkleTreeBuilderList();
+ chunkMerkleTreeBuilderList.get(chunkIndex).setIsHealthy(false).setDataChecksum(random.nextLong());
+ blockMerkleTreeBuilder.setDataChecksum(random.nextLong());
+ }
+ builder.getContainerMerkleTreeBuilder().addBlockMerkleTree(blockMerkleTreeBuilder.build());
+ }
+ builder.getContainerMerkleTreeBuilder().setDataChecksum(random.nextLong());
+ Files.deleteIfExists(getContainerChecksumFile(keyValueContainer.getContainerData()).toPath());
+ writeContainerDataTreeProto(keyValueContainer.getContainerData(), builder.getContainerMerkleTree());
+ }
+ }
+
+ /**
+ * Overwrite the file with random bytes at an offset within the given length.
+ */
+ public static void corruptFileAtOffset(File file, int offset, int chunkLength) {
+ try {
+ final int fileLength = (int) file.length();
+ assertTrue(fileLength >= offset + chunkLength);
+ final int chunkEnd = offset + chunkLength;
+
+ Path path = file.toPath();
+ final byte[] original = IOUtils.readFully(Files.newInputStream(path), fileLength);
+
+ // Corrupt the last byte and middle bytes of the block. The scanner should log this as two errors.
+ final byte[] corruptedBytes = Arrays.copyOf(original, fileLength);
+ corruptedBytes[chunkEnd - 1] = (byte) (original[chunkEnd - 1] << 1);
+ final long chunkMid = offset + ((long) chunkLength - offset) / 2;
+ corruptedBytes[(int) (chunkMid / 2)] = (byte) (original[(int) (chunkMid / 2)] << 1);
+
+
+ Files.write(path, corruptedBytes,
+ StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.SYNC);
+
+ assertThat(IOUtils.readFully(Files.newInputStream(path), fileLength))
+ .isEqualTo(corruptedBytes)
+ .isNotEqualTo(original);
+ } catch (IOException ex) {
+ // Fail the test.
+ throw new UncheckedIOException(ex);
+ }
+ }
}
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
index dd557beff3,34a1d68a1f..343c8dcfee
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
@@@ -74,15 -66,11 +72,13 @@@ import org.junit.jupiter.params.provide
* container is unhealthy.
*/
public class TestKeyValueHandlerWithUnhealthyContainer {
- public static final Logger LOG = LoggerFactory.getLogger(
- TestKeyValueHandlerWithUnhealthyContainer.class);
- @TempDir
- private File tempDir;
-
private IncrementalReportSender<Container> mockIcrSender;
+ private OzoneConfiguration conf;
+ @TempDir
+ private Path tempDir;
+ @TempDir
+ private Path dbFile;
@BeforeEach
public void init() {
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java
index 16e9968b38,075dc6e865..6265ac7d3b
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java
@@@ -21,6 -20,6 +21,7 @@@ import static java.nio.charset.Standard
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getChunk;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.setDataChecksum;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.WRITE_STAGE;
++import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@@ -152,176 -131,6 +153,175 @@@ public class TestFilePerBlockStrategy e
assertEquals(data.rewind().toByteString().substring(start, start + length), readData2.toByteString());
}
+ @ParameterizedTest
+ @MethodSource("getNonClosedStates")
+ public void testWriteChunkAndPutBlockFailureForNonClosedContainer(
+ ContainerProtos.ContainerDataProto.State state) throws IOException {
+ KeyValueContainer keyValueContainer = getKeyValueContainer();
+ keyValueContainer.getContainerData().setState(state);
- ContainerSet containerSet = new ContainerSet(100);
++ ContainerSet containerSet = newContainerSet();
+ containerSet.addContainer(keyValueContainer);
+ KeyValueHandler keyValueHandler = createKeyValueHandler(containerSet);
+ ChunkBuffer.wrap(getData());
+ Assertions.assertThrows(IOException.class, () -> keyValueHandler.writeChunkForClosedContainer(
+ getChunkInfo(), getBlockID(), ChunkBuffer.wrap(getData()), keyValueContainer));
+ Assertions.assertThrows(IOException.class, () -> keyValueHandler.putBlockForClosedContainer(keyValueContainer,
+ new BlockData(getBlockID()), 0L, true));
+ }
+
+ @Test
+ public void testWriteChunkForClosedContainer()
+ throws IOException {
+ ChunkBuffer writeChunkData = ChunkBuffer.wrap(getData());
+ KeyValueContainer kvContainer = getKeyValueContainer();
+ KeyValueContainerData containerData = kvContainer.getContainerData();
+ closedKeyValueContainer();
- ContainerSet containerSet = new ContainerSet(100);
++ ContainerSet containerSet = newContainerSet();
+ containerSet.addContainer(kvContainer);
+ KeyValueHandler keyValueHandler = createKeyValueHandler(containerSet);
+ keyValueHandler.writeChunkForClosedContainer(getChunkInfo(), getBlockID(), writeChunkData, kvContainer);
+ ChunkBufferToByteString readChunkData = keyValueHandler.getChunkManager().readChunk(kvContainer,
+ getBlockID(), getChunkInfo(), WRITE_STAGE);
+ rewindBufferToDataStart();
+ Assertions.assertEquals(writeChunkData, readChunkData);
+ Assertions.assertEquals(containerData.getWriteBytes(), writeChunkData.remaining());
+ Assertions.assertEquals(containerData.getBytesUsed(), writeChunkData.remaining());
+
+ // Test Overwrite
+ keyValueHandler.writeChunkForClosedContainer(getChunkInfo(), getBlockID(),
+ writeChunkData, kvContainer);
+ readChunkData = keyValueHandler.getChunkManager().readChunk(kvContainer,
+ getBlockID(), getChunkInfo(), WRITE_STAGE);
+ rewindBufferToDataStart();
+ Assertions.assertEquals(writeChunkData, readChunkData);
+ Assertions.assertEquals(containerData.getWriteBytes(), 2L * writeChunkData.remaining());
+ // Overwrites won't increase the bytesUsed of a Container
+ Assertions.assertEquals(containerData.getBytesUsed(), writeChunkData.remaining());
+
+ // Test new write chunk after overwrite
+ byte[] bytes = "testing write chunks with after overwrite".getBytes(UTF_8);
+ ChunkBuffer newWriteChunkData = ChunkBuffer.allocate(bytes.length).put(bytes);
+ newWriteChunkData.rewind();
+
+ // Write chunk after the previous overwrite chunk.
+ ChunkInfo newChunkInfo = new ChunkInfo(String.format("%d.data.%d", getBlockID()
+ .getLocalID(), writeChunkData.remaining()), writeChunkData.remaining(), bytes.length);
+ keyValueHandler.writeChunkForClosedContainer(newChunkInfo, getBlockID(),
+ newWriteChunkData, kvContainer);
+ readChunkData = keyValueHandler.getChunkManager().readChunk(kvContainer,
+ getBlockID(), newChunkInfo, WRITE_STAGE);
+ newWriteChunkData.rewind();
+ Assertions.assertEquals(newWriteChunkData, readChunkData);
+ Assertions.assertEquals(containerData.getWriteBytes(), 2L * writeChunkData.remaining()
+ + newWriteChunkData.remaining());
+ Assertions.assertEquals(containerData.getBytesUsed(), writeChunkData.remaining() + newWriteChunkData.remaining());
+ }
+
+ @Test
+ public void testPutBlockForClosedContainer() throws IOException {
+ KeyValueContainer kvContainer = getKeyValueContainer();
+ KeyValueContainerData containerData = kvContainer.getContainerData();
+ closedKeyValueContainer();
- ContainerSet containerSet = new ContainerSet(100);
++ ContainerSet containerSet = newContainerSet();
+ containerSet.addContainer(kvContainer);
+ KeyValueHandler keyValueHandler = createKeyValueHandler(containerSet);
+ List<ContainerProtos.ChunkInfo> chunkInfoList = new ArrayList<>();
+ ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", getBlockID().getLocalID(), 0), 0L, 20L);
+
+ chunkInfoList.add(info.getProtoBufMessage());
+ BlockData putBlockData = new BlockData(getBlockID());
+ putBlockData.setChunks(chunkInfoList);
+
+ ChunkBuffer chunkData = ContainerTestHelper.getData(20);
+ keyValueHandler.writeChunkForClosedContainer(info, getBlockID(), chunkData, kvContainer);
+ keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 1L, true);
+ assertEquals(1L, containerData.getBlockCommitSequenceId());
+ assertEquals(1L, containerData.getBlockCount());
+ assertEquals(20L, containerData.getBytesUsed());
+
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) {
+ long localID = putBlockData.getLocalID();
+ BlockData getBlockData = dbHandle.getStore().getBlockDataTable()
+ .get(containerData.getBlockKey(localID));
+ Assertions.assertTrue(blockDataEquals(putBlockData, getBlockData));
+ assertEquals(20L, dbHandle.getStore().getMetadataTable().get(containerData.getBytesUsedKey()));
+ }
+
+ // Add another chunk and check the put block data
+ ChunkInfo newChunkInfo = new ChunkInfo(String.format("%d.data.%d", getBlockID().getLocalID(), 1L), 20L, 20L);
+ chunkInfoList.add(newChunkInfo.getProtoBufMessage());
+ putBlockData.setChunks(chunkInfoList);
+
+ chunkData = ContainerTestHelper.getData(20);
+ keyValueHandler.writeChunkForClosedContainer(newChunkInfo, getBlockID(), chunkData, kvContainer);
+ keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 2L, true);
+ assertEquals(2L, containerData.getBlockCommitSequenceId());
+ assertEquals(1L, containerData.getBlockCount());
+ assertEquals(40L, containerData.getBytesUsed());
+
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) {
+ long localID = putBlockData.getLocalID();
+ BlockData getBlockData = dbHandle.getStore().getBlockDataTable()
+ .get(containerData.getBlockKey(localID));
+ Assertions.assertTrue(blockDataEquals(putBlockData, getBlockData));
+ assertEquals(40L, dbHandle.getStore().getMetadataTable().get(containerData.getBytesUsedKey()));
+ }
+
+ // Replace the last chunk with a chunk of greater size, This should only update the bytesUsed with
+ // difference in length between the old last chunk and new last chunk
+ newChunkInfo = new ChunkInfo(String.format("%d.data.%d", getBlockID().getLocalID(), 1L), 20L, 30L);
+ chunkInfoList.remove(chunkInfoList.size() - 1);
+ chunkInfoList.add(newChunkInfo.getProtoBufMessage());
+ putBlockData.setChunks(chunkInfoList);
+
+ chunkData = ContainerTestHelper.getData(30);
+ keyValueHandler.writeChunkForClosedContainer(newChunkInfo, getBlockID(), chunkData, kvContainer);
+ keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 2L, true);
+ assertEquals(2L, containerData.getBlockCommitSequenceId());
+ assertEquals(1L, containerData.getBlockCount());
+ // Old chunk size 20, new chunk size 30, difference 10. So bytesUsed should be 40 + 10 = 50
+ assertEquals(50L, containerData.getBytesUsed());
+
+ try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) {
+ long localID = putBlockData.getLocalID();
+ BlockData getBlockData = dbHandle.getStore().getBlockDataTable()
+ .get(containerData.getBlockKey(localID));
+ Assertions.assertTrue(blockDataEquals(putBlockData, getBlockData));
+ assertEquals(50L, dbHandle.getStore().getMetadataTable().get(containerData.getBytesUsedKey()));
+ }
+
+ keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 2L, true);
+ assertEquals(2L, containerData.getBlockCommitSequenceId());
+ }
+
+ private boolean blockDataEquals(BlockData putBlockData, BlockData getBlockData) {
+ return getBlockData.getSize() == putBlockData.getSize() &&
+ Objects.equals(getBlockData.getBlockID(), putBlockData.getBlockID()) &&
+ Objects.equals(getBlockData.getMetadata(), putBlockData.getMetadata()) &&
+ Objects.equals(getBlockData.getChunks(), putBlockData.getChunks());
+ }
+
-
+ private static Stream<Arguments> getNonClosedStates() {
+ return Stream.of(
+ Arguments.of(ContainerProtos.ContainerDataProto.State.OPEN),
+ Arguments.of(ContainerProtos.ContainerDataProto.State.RECOVERING),
+ Arguments.of(ContainerProtos.ContainerDataProto.State.CLOSING),
+ Arguments.of(ContainerProtos.ContainerDataProto.State.INVALID));
+ }
+
+ public KeyValueHandler createKeyValueHandler(ContainerSet containerSet)
+ throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ String dnUuid = UUID.randomUUID().toString();
+ MutableVolumeSet volumeSet = new MutableVolumeSet(dnUuid, conf,
+ null, StorageVolume.VolumeType.DATA_VOLUME, null);
+ return ContainerTestUtils.getKeyValueHandler(conf, dnUuid, containerSet, volumeSet);
+ }
+
+ public void closedKeyValueContainer() {
+ getKeyValueContainer().getContainerData().setState(ContainerProtos.ContainerDataProto.State.CLOSED);
+ }
+
@Override
protected ContainerLayoutTestInfo getStrategy() {
return ContainerLayoutTestInfo.FILE_PER_BLOCK;
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java
index a38c809628,93f8a957e7..8c46b8d8f6
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java
@@@ -18,9 -18,9 +18,10 @@@
package org.apache.hadoop.ozone.container.ozoneimpl;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
-import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.getUnhealthyScanResult;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.getHealthyMetadataScanResult;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.getUnhealthyDataScanResult;
import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java
index b349507488,c4e82714ed..9e684736f9
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java
@@@ -18,9 -18,9 +18,10 @@@
package org.apache.hadoop.ozone.container.ozoneimpl;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
-import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.getUnhealthyScanResult;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.getHealthyDataScanResult;
+import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.getUnhealthyMetadataScanResult;
import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@@ -180,7 -182,8 +182,8 @@@ public class TestBackgroundContainerMet
GenericTestUtils.waitFor(() -> metrics.getNumScanIterations() >= 1, 1000,
5000);
// Volume health should have been checked.
- verify(vol, atLeastOnce()).isFailed();
- // TODO: remove the mock return value asseration after we upgrade to spotbugs 4.8 up
++ // TODO: remove the mock return value assertion after we upgrade to spotbugs 4.8 up
+ assertFalse(verify(vol, atLeastOnce()).isFailed());
// Scanner should not have shutdown when it encountered the failed volume.
assertTrue(scanner.isAlive());
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestContainerImporter.java
index a61c4835fb,8b9d0d1d6d..e6fe1e05da
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestContainerImporter.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestContainerImporter.java
@@@ -150,10 -158,11 +157,10 @@@ class TestContainerImporter
KeyValueContainerData containerData = spy(new KeyValueContainerData(containerId,
ContainerLayoutVersion.FILE_PER_BLOCK, 100, "test", "test"));
// mock to return different checksum
- when(containerData.getChecksum()).thenReturn("checksum1", "checksum2");
- doNothing().when(containerData).setChecksumTo0ByteArray();
+ when(containerData.getContainerFileChecksum()).thenReturn("checksum1", "checksum2");
// create containerImporter object
ContainerController controllerMock = mock(ContainerController.class);
- ContainerSet containerSet = new ContainerSet(0);
+ ContainerSet containerSet = newContainerSet(0);
MutableVolumeSet volumeSet = new MutableVolumeSet("test", conf, null,
StorageVolume.VolumeType.DATA_VOLUME, null);
ContainerImporter containerImporter = spy(new ContainerImporter(conf,
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
index c4bf8a6919,9acb73486a..3ee21a66a0
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@@ -71,10 -82,12 +84,14 @@@ import org.apache.hadoop.hdds.scm.ScmCo
import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl;
+import org.apache.hadoop.ozone.container.checksum.DNContainerOperationClient;
+import org.apache.hadoop.ozone.container.checksum.ReconcileContainerTask;
+ import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+ import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+ import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+ import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
@@@ -126,9 -144,9 +149,11 @@@ public class TestReplicationSupervisor
private StateContext context;
private TestClock clock;
private DatanodeDetails datanode;
+ private DNContainerOperationClient mockClient;
+ private ContainerController mockController;
+ private VolumeChoosingPolicy volumeChoosingPolicy;
+
@BeforeEach
public void setUp() throws Exception {
clock = new TestClock(Instant.now(), ZoneId.systemDefault());
@@@ -140,9 -158,8 +165,10 @@@
stateMachine, "");
context.setTermOfLeaderSCM(CURRENT_TERM);
datanode = MockDatanodeDetails.randomDatanodeDetails();
+ mockClient = mock(DNContainerOperationClient.class);
+ mockController = mock(ContainerController.class);
when(stateMachine.getDatanodeDetails()).thenReturn(datanode);
+ volumeChoosingPolicy = VolumeChoosingPolicyFactory.getPolicy(new OzoneConfiguration());
}
@AfterEach
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
index 0e2baeeeb3,43267d4265..aee25dbbb4
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
@@@ -53,9 -52,12 +53,13 @@@ public final class ContainerReplica imp
replicaIndex = b.replicaIndex;
isEmpty = b.isEmpty;
sequenceId = b.sequenceId;
+ dataChecksum = b.dataChecksum;
}
+ public ContainerID getContainerID() {
+ return containerID;
+ }
+
/**
* Returns the DatanodeDetails to which this replica belongs.
*
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 05f9b6050a,797a6dfd61..ec24d29c16
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@@ -212,13 -212,10 +212,17 @@@ public final class SCMEvents
new TypedEvent<>(CommandStatusReportHandler.DeleteBlockStatus.class,
"Delete_Block_Status");
+ public static final TypedEvent<DatanodeDetails>
+ REPLICATION_MANAGER_NOTIFY =
+ new TypedEvent<>(DatanodeDetails.class, "Replication_Manager_Notify");
+
+ /**
+ * This event will be triggered whenever a datanode needs to reconcile its replica of a container with other
+ * replicas in the cluster.
+ */
+ public static final TypedEvent<ContainerID>
+ RECONCILE_CONTAINER = new TypedEvent<>(ContainerID.class, "Reconcile_Container");
+
/**
* Private Ctor. Never Constructed.
*/
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index e6a74b395f,cd64d669aa..afe74884f5
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@@ -282,9 -272,9 +272,8 @@@ public interface NodeManager extends St
* @param dnId datanode uuid
* @param command
*/
- void addDatanodeCommand(UUID dnId, SCMCommand command);
-
+ void addDatanodeCommand(UUID dnId, SCMCommand<?> command);
-
/**
* send refresh command to all the healthy datanodes to refresh
* volume usage info immediately.
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 07c0a66598,7bc437627a..4c2e102ab7
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@@ -328,25 -336,36 +338,38 @@@ public class SCMClientProtocolServer im
public List<HddsProtos.SCMContainerReplicaProto> getContainerReplicas(
long containerId, int clientVersion) throws IOException {
List<HddsProtos.SCMContainerReplicaProto> results = new ArrayList<>();
+ Map<String, String> auditMap = new HashMap<>();
+ auditMap.put("containerId", String.valueOf(containerId));
+ auditMap.put("clientVersion", String.valueOf(clientVersion));
- Set<ContainerReplica> replicas = getScm().getContainerManager()
- .getContainerReplicas(ContainerID.valueOf(containerId));
- for (ContainerReplica r : replicas) {
- results.add(
- HddsProtos.SCMContainerReplicaProto.newBuilder()
- .setContainerID(containerId)
- .setState(r.getState().toString())
- .setDatanodeDetails(r.getDatanodeDetails().toProto(clientVersion))
- .setBytesUsed(r.getBytesUsed())
- .setPlaceOfBirth(r.getOriginDatanodeId().toString())
- .setKeyCount(r.getKeyCount())
- .setSequenceID(r.getSequenceId())
- .setReplicaIndex(r.getReplicaIndex())
- .setDataChecksum(r.getDataChecksum())
- .build()
- );
+ try {
+ Set<ContainerReplica> replicas = getScm().getContainerManager()
+ .getContainerReplicas(ContainerID.valueOf(containerId));
+ for (ContainerReplica r : replicas) {
+ results.add(
+ HddsProtos.SCMContainerReplicaProto.newBuilder()
+ .setContainerID(containerId)
+ .setState(r.getState().toString())
+ .setDatanodeDetails(r.getDatanodeDetails().toProto(clientVersion))
+ .setBytesUsed(r.getBytesUsed())
+ .setPlaceOfBirth(r.getOriginDatanodeId().toString())
+ .setKeyCount(r.getKeyCount())
+ .setSequenceID(r.getSequenceId())
- .setReplicaIndex(r.getReplicaIndex()).build()
++ .setReplicaIndex(r.getReplicaIndex())
++ .setDataChecksum(r.getDataChecksum())
++ .build()
+ );
+ }
+ AUDIT.logReadSuccess(buildAuditMessageForSuccess(
+ SCMAction.GET_CONTAINER_WITH_PIPELINE_BATCH,
+ auditMap));
+ return results;
+ } catch (Exception ex) {
+ AUDIT.logReadFailure(buildAuditMessageForFailure(
+ SCMAction.GET_CONTAINER_REPLICAS,
+ auditMap, ex));
+ throw ex;
}
- return results;
}
@Override
@@@ -1446,45 -1619,18 +1623,56 @@@
@Override
public String getMetrics(String query) throws IOException {
- FetchMetrics fetchMetrics = new FetchMetrics();
- return fetchMetrics.getMetrics(query);
+ final Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("query", query);
+ try {
+ FetchMetrics fetchMetrics = new FetchMetrics();
+ String metrics = fetchMetrics.getMetrics(query);
+ AUDIT.logReadSuccess(buildAuditMessageForSuccess(
+ SCMAction.GET_METRICS, auditMap));
+ return metrics;
+ } catch (Exception ex) {
+ AUDIT.logReadFailure(buildAuditMessageForFailure(
+ SCMAction.GET_METRICS, auditMap, ex));
+ throw ex;
+ }
}
+
+ @Override
+ public void reconcileContainer(long longContainerID) throws IOException {
+ ContainerID containerID = ContainerID.valueOf(longContainerID);
+ getScm().checkAdminAccess(getRemoteUser(), false);
+ final UserGroupInformation remoteUser = getRemoteUser();
+ final Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("containerID", containerID.toString());
+ auditMap.put("remoteUser", remoteUser.getUserName());
+
+ try {
+ EligibilityResult result = ReconciliationEligibilityHandler.isEligibleForReconciliation(containerID,
+ getScm().getContainerManager());
+ if (!result.isOk()) {
+ switch (result.getResult()) {
+ case OK:
+ break;
+ case CONTAINER_NOT_FOUND:
+ throw new ContainerNotFoundException(result.toString());
+ case INELIGIBLE_CONTAINER_STATE:
+ throw new SCMException(result.toString(), ResultCodes.UNEXPECTED_CONTAINER_STATE);
+ case INELIGIBLE_REPLICA_STATES:
+ case INELIGIBLE_REPLICATION_TYPE:
+ case NOT_ENOUGH_REQUIRED_NODES:
+ case NO_REPLICAS_FOUND:
+ throw new SCMException(result.toString(), ResultCodes.UNSUPPORTED_OPERATION);
+ default:
+ throw new SCMException("Unknown reconciliation eligibility result " + result, ResultCodes.INTERNAL_ERROR);
+ }
+ }
+
+ scm.getEventQueue().fireEvent(SCMEvents.RECONCILE_CONTAINER, containerID);
+ AUDIT.logWriteSuccess(buildAuditMessageForSuccess(SCMAction.RECONCILE_CONTAINER, auditMap));
+ } catch (SCMException ex) {
+ AUDIT.logWriteFailure(buildAuditMessageForFailure(SCMAction.RECONCILE_CONTAINER, auditMap, ex));
+ throw ex;
+ }
+ }
}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index f859367490,04bf80366e..5cfa4e42c9
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@@ -496,11 -494,11 +495,14 @@@ public final class StorageContainerMana
new IncrementalContainerReportHandler(
scmNodeManager, containerManager, scmContext);
PipelineActionHandler pipelineActionHandler =
- new PipelineActionHandler(pipelineManager, scmContext, configuration);
+ new PipelineActionHandler(pipelineManager, scmContext);
+
+ ReplicationManagerEventHandler replicationManagerEventHandler =
+ new ReplicationManagerEventHandler(replicationManager, scmContext);
+ ReconcileContainerEventHandler reconcileContainerEventHandler =
+ new ReconcileContainerEventHandler(containerManager, scmContext);
+
eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler);
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
index 5c78082dec,a514bda0d4..c8e1351297
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
@@@ -55,7 -55,20 +55,21 @@@ public enum SCMAction implements AuditA
RESET_DELETED_BLOCK_RETRY_COUNT,
TRANSFER_LEADERSHIP,
GET_FAILED_DELETED_BLOCKS_TRANSACTION,
+ GET_CONTAINER_REPLICAS,
+ GET_CONTAINERS_ON_DECOM_NODE,
+ DECOMMISSION_NODES,
+ START_MAINTENANCE_NODES,
+ GET_SAFE_MODE_RULE_STATUSES,
+ FINALIZE_SCM_UPGRADE,
+ QUERY_UPGRADE_FINALIZATION_PROGRESS,
+ GET_DATANODE_USAGE_INFO,
+ GET_CONTAINER_TOKEN,
+ GET_CONTAINER_COUNT,
+ DECOMMISSION_SCM,
+ GET_METRICS,
+ QUERY_NODE,
- GET_PIPELINE;
++ GET_PIPELINE,
+ RECONCILE_CONTAINER;
@Override
public String getAction() {
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
index 44c288404c,629ef75916..58967ab918
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
@@@ -23,7 -23,8 +23,9 @@@ import static org.apache.hadoop.hdds.sc
import static org.apache.hadoop.hdds.scm.HddsTestUtils.getECContainer;
import static org.apache.hadoop.hdds.scm.HddsTestUtils.getReplicas;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+ import static org.junit.jupiter.api.Assertions.fail;
+ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
@@@ -1076,159 -1166,16 +1168,163 @@@ public class TestContainerReportHandle
new ContainerReportFromDatanode(datanodeOne, containerReport);
reportHandler.onMessage(containerReportFromDatanode, publisher);
- verify(publisher, times(1)).fireEvent(any(), any(CommandForDatanode.class));
-
- assertEquals(0, containerManager.getContainerReplicas(
- containerOne.containerID()).size());
+ if (isEmpty) {
+ // Expect the replica to be deleted when it is empty
+ verify(publisher, times(1)).fireEvent(any(), any(CommandForDatanode.class));
+ } else {
+ // Expect the replica to stay when it is NOT empty
+ verify(publisher, times(0)).fireEvent(any(), any(CommandForDatanode.class));
+ }
+ assertEquals(1, containerManager.getContainerReplicas(containerOne.containerID()).size());
}
+ @Test
+ public void testWithNoContainerDataChecksum() throws Exception {
+ final ContainerReportHandler reportHandler = new ContainerReportHandler(nodeManager, containerManager);
+
+ final int numNodes = 3;
+ List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeStatus.inServiceHealthy()).stream()
+ .limit(numNodes)
+ .collect(Collectors.toList());
+
+ // Create a container and put one replica on each datanode.
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ ContainerID contID = container.containerID();
+ final Set<ContainerID> containerIDSet = Stream.of(contID).collect(Collectors.toSet());
+
+ for (DatanodeDetails dn: datanodes) {
+ nodeManager.setContainers(dn, containerIDSet);
+ }
+
+ containerStateManager.addContainer(container.getProtobuf());
+
+ getReplicas(contID, ContainerReplicaProto.State.CLOSED, 0, datanodes)
- .forEach(r -> containerStateManager.updateContainerReplica(contID, r));
++ .forEach(containerStateManager::updateContainerReplica);
+
+ // Container manager should now be aware of 3 replicas of each container.
+ assertEquals(numNodes, containerManager.getContainerReplicas(contID).size());
+
+ // All replicas should start with an empty data checksum in SCM.
+ boolean contOneDataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream()
+ .allMatch(r -> r.getDataChecksum() == 0);
+ assertTrue(contOneDataChecksumsEmpty, "Replicas of container one should not yet have any data checksums.");
+
+ // Send a report to SCM from one datanode that still does not have a data checksum.
+ int numReportsSent = 0;
+ for (DatanodeDetails dn: datanodes) {
+ final ContainerReportsProto dnReportProto = getContainerReportsProto(
+ contID, ContainerReplicaProto.State.CLOSED, dn.getUuidString());
+ final ContainerReportFromDatanode dnReport = new ContainerReportFromDatanode(dn, dnReportProto);
+ reportHandler.onMessage(dnReport, publisher);
+ numReportsSent++;
+ }
+ assertEquals(numNodes, numReportsSent);
+
+ // Regardless of which datanode sent the report, none of them have checksums, so all replica's data checksums
+ // should remain empty.
+ boolean containerDataChecksumEmpty = containerManager.getContainerReplicas(contID).stream()
+ .allMatch(r -> r.getDataChecksum() == 0);
+ assertTrue(containerDataChecksumEmpty, "Replicas of the container should not have any data checksums.");
+ }
+
+ @Test
+ public void testWithContainerDataChecksum() throws Exception {
+ final ContainerReportHandler reportHandler = new ContainerReportHandler(nodeManager, containerManager);
+
+ final int numNodes = 3;
+ List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeStatus.inServiceHealthy()).stream()
+ .limit(numNodes)
+ .collect(Collectors.toList());
+
+ // Create a container and put one replica on each datanode.
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ ContainerID contID = container.containerID();
+ final Set<ContainerID> containerIDSet = Stream.of(contID).collect(Collectors.toSet());
+
+ for (DatanodeDetails dn: datanodes) {
+ nodeManager.setContainers(dn, containerIDSet);
+ }
+
+ containerStateManager.addContainer(container.getProtobuf());
+
+ getReplicas(contID, ContainerReplicaProto.State.CLOSED, 0, datanodes)
- .forEach(r -> containerStateManager.updateContainerReplica(contID, r));
++ .forEach(containerStateManager::updateContainerReplica);
+
+ // Container manager should now be aware of 3 replicas of each container.
+ assertEquals(numNodes, containerManager.getContainerReplicas(contID).size());
+
+ // All replicas should start with an empty data checksum in SCM.
+ boolean dataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream()
+ .allMatch(r -> r.getDataChecksum() == 0);
+ assertTrue(dataChecksumsEmpty, "Replicas of container one should not yet have any data checksums.");
+
+ // For each datanode, send a container report with a mismatched checksum.
+ for (DatanodeDetails dn: datanodes) {
+ ContainerReportsProto dnReportProto = getContainerReportsProto(
+ contID, ContainerReplicaProto.State.CLOSED, dn.getUuidString());
+ ContainerReplicaProto replicaWithChecksum = dnReportProto.getReports(0).toBuilder()
+ .setDataChecksum(createUniqueDataChecksumForReplica(contID, dn.getUuidString()))
+ .build();
+ ContainerReportsProto reportWithChecksum = dnReportProto.toBuilder()
+ .clearReports()
+ .addReports(replicaWithChecksum)
+ .build();
+ final ContainerReportFromDatanode dnReport = new ContainerReportFromDatanode(dn, reportWithChecksum);
+ reportHandler.onMessage(dnReport, publisher);
+ }
+
+ // All the replicas should have different checksums.
+ // Since the containers don't have any data in this test, different checksums are based on container ID and
+ // datanode ID.
+ int numReplicasChecked = 0;
+ for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) {
+ long expectedChecksum = createUniqueDataChecksumForReplica(contID, replica.getDatanodeDetails().getUuidString());
+ assertEquals(expectedChecksum, replica.getDataChecksum());
+ numReplicasChecked++;
+ }
+ assertEquals(numNodes, numReplicasChecked);
+
+ // For each datanode, send a container report with a matching checksum.
+ // This simulates reconciliation running.
+ for (DatanodeDetails dn: datanodes) {
+ ContainerReportsProto dnReportProto = getContainerReportsProto(
+ contID, ContainerReplicaProto.State.CLOSED, dn.getUuidString());
+ ContainerReplicaProto replicaWithChecksum = dnReportProto.getReports(0).toBuilder()
+ .setDataChecksum(createMatchingDataChecksumForReplica(contID))
+ .build();
+ ContainerReportsProto reportWithChecksum = dnReportProto.toBuilder()
+ .clearReports()
+ .addReports(replicaWithChecksum)
+ .build();
+ final ContainerReportFromDatanode dnReport = new ContainerReportFromDatanode(dn, reportWithChecksum);
+ reportHandler.onMessage(dnReport, publisher);
+ }
+
+ // All the replicas should now have matching checksums.
+ // Since the containers don't have any data in this test, the matching checksums are based on container ID only.
+ numReplicasChecked = 0;
+ for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) {
+ long expectedChecksum = createMatchingDataChecksumForReplica(contID);
+ assertEquals(expectedChecksum, replica.getDataChecksum());
+ numReplicasChecked++;
+ }
+ assertEquals(numNodes, numReplicasChecked);
+ }
+
+ /**
+ * Generates a placeholder data checksum for testing that is specific to a container replica.
+ */
+ protected static long createUniqueDataChecksumForReplica(ContainerID containerID, String datanodeID) {
+ return (datanodeID + containerID).hashCode();
+ }
+
+ /**
+ * Generates a placeholder data checksum for testing that is the same for all container replicas.
+ */
+ protected static long createMatchingDataChecksumForReplica(ContainerID containerID) {
+ return Objects.hashCode(containerID);
+ }
+
private ContainerReportFromDatanode getContainerReportFromDatanode(
ContainerID containerId, ContainerReplicaProto.State state,
DatanodeDetails dn, long bytesUsed, long keyCount) {
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
index 8dba53e148,976c4d6978..07af128809
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
@@@ -29,7 -27,7 +29,8 @@@ import static org.apache.hadoop.hdds.sc
import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+ import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
@@@ -575,144 -618,6 +621,140 @@@ public class TestIncrementalContainerRe
}
}
+ @Test
+ public void testWithNoContainerDataChecksum() throws Exception {
+ final IncrementalContainerReportHandler reportHandler = new IncrementalContainerReportHandler(nodeManager,
+ containerManager, scmContext);
+
+ final int numNodes = 3;
+
+ // Create a container which will have one replica on each datanode.
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ ContainerID contID = container.containerID();
- final Set<ContainerID> containerIDSet = Stream.of(contID).collect(Collectors.toSet());
+
+ List<DatanodeDetails> datanodes = new ArrayList<>();
+ for (int i = 0; i < numNodes; i++) {
+ DatanodeDetails dn = randomDatanodeDetails();
+ nodeManager.register(dn, null, null);
- nodeManager.setContainers(dn, containerIDSet);
+ datanodes.add(dn);
+ }
+
+ containerStateManager.addContainer(container.getProtobuf());
+
+ getReplicas(contID, ContainerReplicaProto.State.CLOSED, 0, datanodes)
- .forEach(r -> containerStateManager.updateContainerReplica(contID, r));
++ .forEach(containerStateManager::updateContainerReplica);
+
+ // Container manager should now be aware of 3 replicas of each container.
+ assertEquals(numNodes, containerManager.getContainerReplicas(contID).size());
+
+ // All replicas should start with an empty data checksum in SCM.
+ boolean contOneDataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream()
+ .allMatch(r -> r.getDataChecksum() == 0);
+ assertTrue(contOneDataChecksumsEmpty, "Replicas of container one should not yet have any data checksums.");
+
+ // Send a report to SCM from one datanode that still does not have a data checksum.
+ for (DatanodeDetails dn: datanodes) {
+ final IncrementalContainerReportProto dnReportProto = getIncrementalContainerReportProto(
+ contID, ContainerReplicaProto.State.CLOSED, dn.getUuidString());
+ final IncrementalContainerReportFromDatanode dnReport = new IncrementalContainerReportFromDatanode(dn,
+ dnReportProto);
+ reportHandler.onMessage(dnReport, publisher);
+ }
+
+ // Regardless of which datanode sent the report, none of them have checksums, so all replica's data checksums
+ // should remain empty.
+ boolean containerDataChecksumEmpty = containerManager.getContainerReplicas(contID).stream()
+ .allMatch(r -> r.getDataChecksum() == 0);
+ assertTrue(containerDataChecksumEmpty, "Replicas of the container should not have any data checksums.");
+ }
+
+ @Test
+ public void testWithContainerDataChecksum() throws Exception {
+ final IncrementalContainerReportHandler reportHandler = new IncrementalContainerReportHandler(nodeManager,
+ containerManager, scmContext);
+
+ final int numNodes = 3;
+
+ // Create a container which will have one replica on each datanode.
+ final ContainerInfo container = getContainer(LifeCycleState.CLOSED);
+ ContainerID contID = container.containerID();
- final Set<ContainerID> containerIDSet = Stream.of(contID).collect(Collectors.toSet());
+
+ List<DatanodeDetails> datanodes = new ArrayList<>();
+ for (int i = 0; i < numNodes; i++) {
+ DatanodeDetails dn = randomDatanodeDetails();
+ nodeManager.register(dn, null, null);
- nodeManager.setContainers(dn, containerIDSet);
+ datanodes.add(dn);
+ }
+
+ containerStateManager.addContainer(container.getProtobuf());
+
+ getReplicas(contID, ContainerReplicaProto.State.CLOSED, 0, datanodes)
- .forEach(r -> containerStateManager.updateContainerReplica(contID, r));
++ .forEach(containerStateManager::updateContainerReplica);
+
+ // Container manager should now be aware of 3 replicas of each container.
+ assertEquals(3, containerManager.getContainerReplicas(contID).size());
+
+ // All replicas should start with a zero data checksum in SCM.
+ boolean dataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream()
+ .allMatch(r -> r.getDataChecksum() == 0);
+ assertTrue(dataChecksumsEmpty, "Replicas of container one should not yet have any data checksums.");
+
+ // For each datanode, send a container report with a mismatched checksum.
+ for (DatanodeDetails dn: datanodes) {
+ IncrementalContainerReportProto dnReportProto = getIncrementalContainerReportProto(
+ contID, ContainerReplicaProto.State.CLOSED, dn.getUuidString());
+ ContainerReplicaProto replicaWithChecksum = dnReportProto.getReport(0).toBuilder()
+ .setDataChecksum(createUniqueDataChecksumForReplica(contID, dn.getUuidString()))
+ .build();
+ IncrementalContainerReportProto reportWithChecksum = dnReportProto.toBuilder()
+ .clearReport()
+ .addReport(replicaWithChecksum)
+ .build();
+ final IncrementalContainerReportFromDatanode dnReport = new IncrementalContainerReportFromDatanode(dn,
+ reportWithChecksum);
+ reportHandler.onMessage(dnReport, publisher);
+ }
+
+ // All the replicas should have different checksums.
+ // Since the containers don't have any data in this test, different checksums are based on container ID and
+ // datanode ID.
+ int numReplicasChecked = 0;
+ for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) {
+ long expectedChecksum = createUniqueDataChecksumForReplica(
+ contID, replica.getDatanodeDetails().getUuidString());
+ assertEquals(expectedChecksum, replica.getDataChecksum());
+ numReplicasChecked++;
+ }
+ assertEquals(numNodes, numReplicasChecked);
+
+ // For each datanode, send a container report with a matching checksum.
+ // This simulates reconciliation running.
+ for (DatanodeDetails dn: datanodes) {
+ IncrementalContainerReportProto dnReportProto = getIncrementalContainerReportProto(
+ contID, ContainerReplicaProto.State.CLOSED, dn.getUuidString());
+ ContainerReplicaProto replicaWithChecksum = dnReportProto.getReport(0).toBuilder()
+ .setDataChecksum(createMatchingDataChecksumForReplica(contID))
+ .build();
+ IncrementalContainerReportProto reportWithChecksum = dnReportProto.toBuilder()
+ .clearReport()
+ .addReport(replicaWithChecksum)
+ .build();
+ IncrementalContainerReportFromDatanode dnReport = new IncrementalContainerReportFromDatanode(dn,
+ reportWithChecksum);
+ reportHandler.onMessage(dnReport, publisher);
+ }
+
+ // All the replicas should now have matching checksums.
+ // Since the containers don't have any data in this test, the matching checksums are based on container ID only.
+ numReplicasChecked = 0;
+ for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) {
+ long expectedChecksum = createMatchingDataChecksumForReplica(contID);
+ assertEquals(expectedChecksum, replica.getDataChecksum());
+ numReplicasChecked++;
+ }
+ assertEquals(numNodes, numReplicasChecked);
+ }
+
private static IncrementalContainerReportProto
getIncrementalContainerReportProto(ContainerReplicaProto replicaProto) {
final IncrementalContainerReportProto.Builder crBuilder =
diff --cc hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
index 55132123cd,8533c1938e..83c2212978
--- a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
@@@ -97,11 -103,53 +109,46 @@@ List all container
List all containers according to count (batchSize)
${output} = Execute ozone admin container list --all --count 10
Should contain ${output} OPEN
+ Should Start With ${output} [
+ Should End With ${output} ]
List all containers from a particular container ID
- ${output} = Execute ozone admin container list --all --start 1
+ ${output} = Execute ozone admin container list --all --start 2
Should contain ${output} OPEN
+ Should Start With ${output} [
+ Should End With ${output} ]
+
+ Check JSON array parsing
+ ${output} = Execute ozone admin container list
+ Should Start With ${output} [
+ Should Contain ${output} containerID
+ Should End With ${output} ]
+ ${containerIDs} = Execute echo '${output}' | jq -r '.[].containerID'
+ Should Not Be Empty ${containerIDs}
+
+ Check state filtering with JSON array format
+ ${output} = Execute ozone admin container list --state=OPEN
+ Should Start With ${output} [
+ Should End With ${output} ]
+ ${states} = Execute echo '${output}' | jq -r '.[].state'
+ Should Contain ${states} OPEN
+ Should Not Contain ${states} CLOSED
+
+ Check count limit with JSON array format
+ ${output} = Execute ozone admin container create
+ Should contain ${output} is created
+ ${output} = Execute ozone admin container create
+ Should contain ${output} is created
+ ${output} = Execute ozone admin container create
+ Should contain ${output} is created
+ ${output} = Execute ozone admin container create
+ Should contain ${output} is created
+ ${output} = Execute ozone admin container create
+ Should contain ${output} is created
+ ${output} = Execute And Ignore Error ozone admin container list --count 5 2> /dev/null # This logs to error that the list is incomplete
+ ${count} = Execute echo '${output}' | jq -r 'length'
+ Should Be True ${count} == 5
-Close container
- ${container} = Execute ozone admin container list --state OPEN | jq -r '.[] | select(.replicationConfig.replicationFactor == "ONE") | .containerID' | head -1
- Execute ozone admin container close "${container}"
- ${output} = Execute ozone admin container info "${container}"
- Should contain ${output} CLOS
- Wait until keyword succeeds 1min 10sec Container is closed ${container}
-
Incomplete command
${output} = Execute And Ignore Error ozone admin container
Should contain ${output} Missing required subcommand
@@@ -110,8 -158,6 +157,7 @@@
Should contain ${output} create
Should contain ${output} close
Should contain ${output} report
- Should contain ${output} upgrade
+ Should contain ${output} reconcile
#List containers on unknown host
# ${output} = Execute And Ignore Error ozone admin --verbose container list --scm unknown-host
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java
index 8c63ec5caf,381c86d403..8093c634b6
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java
@@@ -17,7 -17,6 +17,7 @@@
package org.apache.hadoop.hdds.scm;
-
++import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
@@@ -27,24 -26,16 +27,25 @@@ import static org.apache.hadoop.hdds.Hd
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.containerChecksumFileExists;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.time.Duration;
+import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
- import org.apache.hadoop.hdds.client.ReplicationFactor;
- import org.apache.hadoop.hdds.client.ReplicationType;
++import org.apache.hadoop.hdds.client.RatisReplicationConfig;
++import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
++import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration;
@@@ -122,8 -106,8 +123,8 @@@ public class TestCloseContainer
throws Exception {
// Create some keys to write data into the open containers
for (int i = 0; i < 10; i++) {
- TestDataUtil.createKey(bucket, "key" + i, ReplicationFactor.THREE,
- ReplicationType.RATIS, "this is the content");
+ TestDataUtil.createKey(bucket, "key" + i,
- "this is the content".getBytes(StandardCharsets.UTF_8));
++ "this is the content".getBytes(UTF_8));
}
StorageContainerManager scm = cluster.getStorageContainerManager();
@@@ -181,8 -151,8 +182,7 @@@
throws Exception {
// Create some keys to write data into the open containers
for (int i = 0; i < 10; i++) {
- TestDataUtil.createKey(bucket, "key" + i, ReplicationFactor.THREE,
- ReplicationType.RATIS, "this is the content");
- TestDataUtil.createKey(bucket, "key" + i,
- "this is the content".getBytes(StandardCharsets.UTF_8));
++ TestDataUtil.createKey(bucket, "key" + i, "this is the content".getBytes(UTF_8));
}
StorageContainerManager scm = cluster.getStorageContainerManager();
// Pick any container on the cluster and close it via client
@@@ -211,83 -164,4 +211,82 @@@
"Container " + container.getContainerID() + " already closed");
}
+ @Test
+ public void testContainerChecksumForClosedContainer() throws Exception {
+ // Create some keys to write data into the open containers
- TestDataUtil.createKey(bucket, "key1", ReplicationFactor.THREE,
- ReplicationType.RATIS, "this is the content");
++ ReplicationConfig repConfig = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE);
++ TestDataUtil.createKey(bucket, "key1", repConfig, "this is the content".getBytes(UTF_8));
+ StorageContainerManager scm = cluster.getStorageContainerManager();
+
+ ContainerInfo containerInfo1 = scm.getContainerManager().getContainers().get(0);
+ // Checksum file doesn't exist before container close
+ List<HddsDatanodeService> hddsDatanodes = cluster.getHddsDatanodes();
+ for (HddsDatanodeService hddsDatanode : hddsDatanodes) {
+ assertFalse(containerChecksumFileExists(hddsDatanode, containerInfo1));
+ }
+ // Close container.
+ OzoneTestUtils.closeContainer(scm, containerInfo1);
+ ContainerProtos.ContainerChecksumInfo prevExpectedChecksumInfo1 = null;
+ // Checksum file exists after container close and matches the expected container
+ // merkle tree for all the datanodes
+ for (HddsDatanodeService hddsDatanode : hddsDatanodes) {
+ GenericTestUtils.waitFor(() -> checkContainerCloseInDatanode(hddsDatanode, containerInfo1), 100, 5000);
+ assertTrue(containerChecksumFileExists(hddsDatanode, containerInfo1));
+ OzoneContainer ozoneContainer = hddsDatanode.getDatanodeStateMachine().getContainer();
+ Container<?> container1 = ozoneContainer.getController().getContainer(containerInfo1.getContainerID());
+ ContainerProtos.ContainerChecksumInfo containerChecksumInfo = ContainerMerkleTreeTestUtils.readChecksumFile(
+ container1.getContainerData());
+ assertNotNull(containerChecksumInfo);
+ if (prevExpectedChecksumInfo1 != null) {
+ ContainerMerkleTreeTestUtils.assertTreesSortedAndMatch(prevExpectedChecksumInfo1.getContainerMerkleTree(),
+ containerChecksumInfo.getContainerMerkleTree());
+ }
+ prevExpectedChecksumInfo1 = containerChecksumInfo;
+ }
+
+ // Create 2nd container and check the checksum doesn't match with 1st container
- TestDataUtil.createKey(bucket, "key2", ReplicationFactor.THREE,
- ReplicationType.RATIS, "this is the different content");
++ TestDataUtil.createKey(bucket, "key2", repConfig, "this is the different content".getBytes());
+ ContainerInfo containerInfo2 = scm.getContainerManager().getContainers().get(1);
+ for (HddsDatanodeService hddsDatanode : hddsDatanodes) {
+ assertFalse(containerChecksumFileExists(hddsDatanode, containerInfo2));
+ }
+
+ // Close container.
+ OzoneTestUtils.closeContainer(scm, containerInfo2);
+ ContainerProtos.ContainerChecksumInfo prevExpectedChecksumInfo2 = null;
+ // Checksum file exists after container close and matches the expected container
+ // merkle tree for all the datanodes
+ for (HddsDatanodeService hddsDatanode : hddsDatanodes) {
+ GenericTestUtils.waitFor(() -> checkContainerCloseInDatanode(hddsDatanode, containerInfo2), 100, 5000);
+ assertTrue(containerChecksumFileExists(hddsDatanode, containerInfo2));
+ OzoneContainer ozoneContainer = hddsDatanode.getDatanodeStateMachine().getContainer();
+ Container<?> container2 = ozoneContainer.getController().getContainer(containerInfo2.getContainerID());
+ ContainerProtos.ContainerChecksumInfo containerChecksumInfo = ContainerMerkleTreeTestUtils.readChecksumFile(
+ container2.getContainerData());
+ assertNotNull(containerChecksumInfo);
+ if (prevExpectedChecksumInfo2 != null) {
+ ContainerMerkleTreeTestUtils.assertTreesSortedAndMatch(prevExpectedChecksumInfo2.getContainerMerkleTree(),
+ containerChecksumInfo.getContainerMerkleTree());
+ }
+ prevExpectedChecksumInfo2 = containerChecksumInfo;
+ }
+
+ // Container merkle tree for different container should not match.
+ assertNotEquals(prevExpectedChecksumInfo1.getContainerID(), prevExpectedChecksumInfo2.getContainerID());
+ assertNotEquals(prevExpectedChecksumInfo1.getContainerMerkleTree().getDataChecksum(),
+ prevExpectedChecksumInfo2.getContainerMerkleTree().getDataChecksum());
+ for (ContainerReplica replica : getContainerReplicas(containerInfo1)) {
+ assertNotEquals(0, replica.getDataChecksum());
+ }
+ for (ContainerReplica replica : getContainerReplicas(containerInfo2)) {
+ assertNotEquals(0, replica.getDataChecksum());
+ }
+ }
+
+ private boolean checkContainerCloseInDatanode(HddsDatanodeService hddsDatanode,
+ ContainerInfo containerInfo) {
+ Container container = hddsDatanode.getDatanodeStateMachine().getContainer().getController()
+ .getContainer(containerInfo.getContainerID());
+ return container.getContainerState() == ContainerProtos.ContainerDataProto.State.CLOSED;
+ }
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
index cfd70a6ca7,4a03bbad8a..5ccbd80fac
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
@@@ -198,8 -198,8 +198,8 @@@ public class TestECKeyOutputStream
OzoneClient client1 = null;
try (MockedStatic<Handler> mockedHandler = Mockito.mockStatic(Handler.class, Mockito.CALLS_REAL_METHODS)) {
Map<String, Handler> handlers = new HashMap<>();
- mockedHandler.when(() -> Handler.getHandlerForContainerType(any(), any(), any(), any(), any(), any(), any(),
- any()))
+ mockedHandler.when(() -> Handler
- .getHandlerForContainerType(any(), any(), any(), any(), any(), any(), any(), any()))
++ .getHandlerForContainerType(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenAnswer(i -> {
Handler handler = Mockito.spy((Handler) i.callRealMethod());
handlers.put(handler.getDatanodeId(), handler);
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
index b083be23ee,fdc2f3feb0..7c322c5143
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
@@@ -47,6 -49,6 +49,7 @@@ import org.apache.hadoop.metrics2.lib.D
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.RatisTestHelper;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
++import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@@@ -130,10 -136,21 +137,21 @@@ public class TestContainerMetrics
}
private HddsDispatcher createDispatcher(DatanodeDetails dd, VolumeSet volumeSet) {
- ContainerSet containerSet = new ContainerSet(1000);
+ ContainerSet containerSet = newContainerSet();
StateContext context = ContainerTestUtils.getMockContext(
dd, CONF);
- HddsDispatcher dispatcher = ContainerTestUtils.getHddsDispatcher(CONF, containerSet, volumeSet, context);
+ ContainerMetrics metrics = ContainerMetrics.create(CONF);
+ Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerProtos.ContainerType containerType :
+ ContainerProtos.ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(containerType, CONF,
+ context.getParent().getDatanodeDetails().getUuidString(),
+ containerSet, volumeSet, volumeChoosingPolicy, metrics,
- c -> { }));
++ c -> { }, new ContainerChecksumTreeManager(CONF)));
+ }
+ HddsDispatcher dispatcher = new HddsDispatcher(CONF, containerSet,
+ volumeSet, handlers, context, metrics, null);
StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList())
.forEach(hddsVolume -> hddsVolume.setDbParentDir(tempDir.toFile()));
dispatcher.setClusterId(UUID.randomUUID().toString());
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
index 05089c33e2,0b3294b913..e2cb7d45fb
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
@@@ -81,11 -74,9 +80,12 @@@ import org.apache.hadoop.hdds.security.
import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier;
import org.apache.hadoop.hdds.security.token.ContainerTokenSecretManager;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl;
+import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.ozone.client.SecretKeyTestClient;
+import org.apache.hadoop.ozone.container.checksum.DNContainerOperationClient;
import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
+import org.apache.hadoop.ozone.container.common.helpers.TokenHelper;
+ import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
index 2808cff2c0,20a45c0242..54644189ec
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
@@@ -51,7 -52,8 +52,9 @@@ import org.apache.hadoop.metrics2.lib.D
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.RatisTestHelper;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
++import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
+ import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@@ -195,7 -201,19 +202,18 @@@ public class TestContainerServer
StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList())
.forEach(hddsVolume -> hddsVolume.setDbParentDir(tempDir.toFile()));
StateContext context = ContainerTestUtils.getMockContext(dd, conf);
- HddsDispatcher hddsDispatcher = ContainerTestUtils.getHddsDispatcher(conf, containerSet, volumeSet, context);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerProtos.ContainerType containerType :
+ ContainerProtos.ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(containerType, conf,
+ dd.getUuid().toString(),
+ containerSet, volumeSet, volumeChoosingPolicy, metrics,
- c -> {
- }));
++ c -> { }, new ContainerChecksumTreeManager(conf)));
+ }
+ HddsDispatcher hddsDispatcher = new HddsDispatcher(
+ conf, containerSet, volumeSet, handlers, context, metrics, null);
hddsDispatcher.setClusterId(scmId.toString());
return hddsDispatcher;
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
index f0cc8f1363,82c2bf2ba3..f02d632602
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
@@@ -81,7 -83,8 +83,9 @@@ import org.apache.hadoop.metrics2.lib.D
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.RatisTestHelper;
import org.apache.hadoop.ozone.client.SecretKeyTestClient;
++import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
import org.apache.hadoop.ozone.container.common.ContainerTestUtils;
+ import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@@ -168,7 -176,18 +177,18 @@@ public class TestSecureContainerServer
StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList())
.forEach(hddsVolume -> hddsVolume.setDbParentDir(tempDir.toFile()));
StateContext context = ContainerTestUtils.getMockContext(dd, conf);
- HddsDispatcher hddsDispatcher = ContainerTestUtils.getHddsDispatcher(conf, containerSet, volumeSet, context,
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerProtos.ContainerType containerType :
+ ContainerProtos.ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(containerType, conf,
+ dd.getUuid().toString(),
+ containerSet, volumeSet, volumeChoosingPolicy, metrics,
- c -> { }));
++ c -> { }, new ContainerChecksumTreeManager(conf)));
+ }
+ HddsDispatcher hddsDispatcher = new HddsDispatcher(
+ conf, containerSet, volumeSet, handlers, context, metrics,
TokenVerifier.create(new SecurityConfig(conf), secretKeyClient));
hddsDispatcher.setClusterId(scmId.toString());
return hddsDispatcher;
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/checksum/TestContainerCommandReconciliation.java
index 4624cc562f,0000000000..d7dfa0bbfe
mode 100644,000000..100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/checksum/TestContainerCommandReconciliation.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/checksum/TestContainerCommandReconciliation.java
@@@ -1,728 -1,0 +1,728 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.dn.checksum;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
- import static org.apache.hadoop.hdds.DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_KEYTAB_FILE_KEY;
- import static org.apache.hadoop.hdds.DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED;
++import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_KERBEROS_KEYTAB_FILE_KEY;
++import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_EXPIRY_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_CHECK_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE;
+import static org.apache.hadoop.hdds.client.ReplicationType.RATIS;
+import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import static org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager.getContainerChecksumFile;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.assertTreesSortedAndMatch;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.buildTestTree;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.readChecksumFile;
+import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.writeContainerDataTreeProto;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfig;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.ozone.ClientVersion;
+import org.apache.hadoop.ozone.HddsDatanodeService;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.container.TestHelper;
+import org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeWriter;
+import org.apache.hadoop.ozone.container.checksum.DNContainerOperationClient;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration;
+import org.apache.hadoop.ozone.container.ozoneimpl.MetadataScanResult;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * This class tests container commands for reconciliation.
+ */
+public class TestContainerCommandReconciliation {
+
+ private static MiniOzoneHAClusterImpl cluster;
+ private static OzoneClient rpcClient;
+ private static ObjectStore store;
+ private static OzoneConfiguration conf;
+ private static DNContainerOperationClient dnClient;
+ private static final String KEY_NAME = "testkey";
+
+ @TempDir
+ private static File testDir;
+ @TempDir
+ private static File workDir;
+ private static MiniKdc miniKdc;
+ private static File ozoneKeytab;
+ private static File spnegoKeytab;
+ private static File testUserKeytab;
+ private static String testUserPrincipal;
+ private static String host;
+
+ @BeforeAll
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost");
+ conf.set(OZONE_METADATA_DIRS, testDir.getAbsolutePath());
+ conf.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, 128 * 1024, StorageUnit.BYTES);
+ conf.setStorageSize(OZONE_SCM_BLOCK_SIZE, 512 * 1024, StorageUnit.BYTES);
+ // Disable the container scanner so it does not create merkle tree files that interfere with this test.
+ // TODO: Currently container scrub sets the checksum to 0, Revert this after HDDS-10374 is merged.
+ conf.getObject(ContainerScannerConfiguration.class).setEnabled(false);
+ conf.setBoolean("hdds.container.scrub.enabled", false);
+
+ startMiniKdc();
+ setSecureConfig();
+ createCredentialsInKDC();
+ setSecretKeysConfig();
+ startCluster();
+ }
+
+ @AfterAll
+ public static void stop() throws IOException {
+ if (rpcClient != null) {
+ rpcClient.close();
+ }
+
+ if (dnClient != null) {
+ dnClient.close();
+ }
+
+ if (miniKdc != null) {
+ miniKdc.stop();
+ }
+
+ if (cluster != null) {
+ cluster.stop();
+ }
+ }
+
+ /**
+ * Container checksum trees are only generated for non-open containers.
+ * Calling the API on a non-open container should fail.
+ */
+ @Test
+ public void testGetChecksumInfoOpenReplica() throws Exception {
+ String volume = UUID.randomUUID().toString();
+ String bucket = UUID.randomUUID().toString();
+ long containerID = writeDataAndGetContainer(false, volume, bucket);
+ HddsDatanodeService targetDN = cluster.getHddsDatanodes().get(0);
+ StorageContainerException ex = assertThrows(StorageContainerException.class,
+ () -> dnClient.getContainerChecksumInfo(containerID, targetDN.getDatanodeDetails()));
+ assertEquals(ex.getResult(), ContainerProtos.Result.UNCLOSED_CONTAINER_IO);
+ }
+
+ /**
+ * Tests reading the container checksum info file from a datanode who does not have a replica for the requested
+ * container.
+ */
+ @Test
+ public void testGetChecksumInfoNonexistentReplica() {
+ HddsDatanodeService targetDN = cluster.getHddsDatanodes().get(0);
+
+ // Find a container ID that does not exist in the cluster. For a small test this should be a good starting
+ // point, but modify it just in case.
+ long badIDCheck = 1_000_000;
+ while (cluster.getStorageContainerManager().getContainerManager()
+ .containerExist(ContainerID.valueOf(badIDCheck))) {
+ badIDCheck++;
+ }
+
+ final long nonexistentContainerID = badIDCheck;
+ StorageContainerException ex = assertThrows(StorageContainerException.class,
+ () -> dnClient.getContainerChecksumInfo(nonexistentContainerID, targetDN.getDatanodeDetails()));
+ assertEquals(ex.getResult(), ContainerProtos.Result.CONTAINER_NOT_FOUND);
+ }
+
+ /**
+ * Tests reading the container checksum info file from a datanode where the container exists, but the file has not
+ * yet been created.
+ */
+ @Test
+ public void testGetChecksumInfoNonexistentFile() throws Exception {
+ String volume = UUID.randomUUID().toString();
+ String bucket = UUID.randomUUID().toString();
+ long containerID = writeDataAndGetContainer(true, volume, bucket);
+ // Pick a datanode and remove its checksum file.
+ HddsDatanodeService targetDN = cluster.getHddsDatanodes().get(0);
+ Container<?> container = targetDN.getDatanodeStateMachine().getContainer()
+ .getContainerSet().getContainer(containerID);
+ File treeFile = getContainerChecksumFile(container.getContainerData());
+ // Closing the container should have generated the tree file.
+ assertTrue(treeFile.exists());
+ assertTrue(treeFile.delete());
+
+ StorageContainerException ex = assertThrows(StorageContainerException.class, () ->
+ dnClient.getContainerChecksumInfo(containerID, targetDN.getDatanodeDetails()));
+ assertEquals(ContainerProtos.Result.IO_EXCEPTION, ex.getResult());
+ assertTrue(ex.getMessage().contains("(No such file or directory"), ex.getMessage() +
+ " did not contain the expected string");
+ }
+
+ /**
+ * Tests reading the container checksum info file from a datanode where the datanode fails to read the file from
+ * the disk.
+ */
+ @Test
+ public void testGetChecksumInfoServerIOError() throws Exception {
+ String volume = UUID.randomUUID().toString();
+ String bucket = UUID.randomUUID().toString();
+ long containerID = writeDataAndGetContainer(true, volume, bucket);
+ // Pick a datanode and remove its checksum file.
+ HddsDatanodeService targetDN = cluster.getHddsDatanodes().get(0);
+ Container<?> container = targetDN.getDatanodeStateMachine().getContainer()
+ .getContainerSet().getContainer(containerID);
+ File treeFile = getContainerChecksumFile(container.getContainerData());
+ assertTrue(treeFile.exists());
+ // Make the server unable to read the file.
+ assertTrue(treeFile.setReadable(false));
+
+ StorageContainerException ex = assertThrows(StorageContainerException.class, () ->
+ dnClient.getContainerChecksumInfo(containerID, targetDN.getDatanodeDetails()));
+ assertEquals(ContainerProtos.Result.IO_EXCEPTION, ex.getResult());
+ }
+
+ /**
+ * Tests reading the container checksum info file from a datanode where the file is corrupt.
+ * The datanode does not deserialize the file before sending it, so there should be no error on the server side
+ * when sending the file. The client should raise an error trying to deserialize it.
+ */
+ @Test
+ public void testGetCorruptChecksumInfo() throws Exception {
+ String volume = UUID.randomUUID().toString();
+ String bucket = UUID.randomUUID().toString();
+ long containerID = writeDataAndGetContainer(true, volume, bucket);
+
+ // Pick a datanode and corrupt its checksum file.
+ HddsDatanodeService targetDN = cluster.getHddsDatanodes().get(0);
+ Container<?> container = targetDN.getDatanodeStateMachine().getContainer()
+ .getContainerSet().getContainer(containerID);
+ File treeFile = getContainerChecksumFile(container.getContainerData());
+ Files.write(treeFile.toPath(), new byte[]{1, 2, 3},
+ StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.SYNC);
+
+ // Reading the file from the replica should fail when the client tries to deserialize it.
+ assertThrows(InvalidProtocolBufferException.class, () -> dnClient.getContainerChecksumInfo(containerID,
+ targetDN.getDatanodeDetails()));
+ }
+
+ @Test
+ public void testGetEmptyChecksumInfo() throws Exception {
+ String volume = UUID.randomUUID().toString();
+ String bucket = UUID.randomUUID().toString();
+ long containerID = writeDataAndGetContainer(true, volume, bucket);
+
+ // Pick a datanode and truncate its checksum file to zero length.
+ HddsDatanodeService targetDN = cluster.getHddsDatanodes().get(0);
+ Container<?> container = targetDN.getDatanodeStateMachine().getContainer()
+ .getContainerSet().getContainer(containerID);
+ File treeFile = getContainerChecksumFile(container.getContainerData());
+ // TODO After HDDS-10379 the file will already exist and need to be overwritten.
+ assertTrue(treeFile.exists());
+ Files.write(treeFile.toPath(), new byte[]{},
+ StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.SYNC);
+ assertEquals(0, treeFile.length());
+
+ // The client will get an empty byte string back. It should raise this as an error instead of returning a default
+ // protobuf object.
+ StorageContainerException ex = assertThrows(StorageContainerException.class, () ->
+ dnClient.getContainerChecksumInfo(containerID, targetDN.getDatanodeDetails()));
+ assertEquals(ContainerProtos.Result.IO_EXCEPTION, ex.getResult());
+ }
+
+ @Test
+ public void testGetChecksumInfoSuccess() throws Exception {
+ String volume = UUID.randomUUID().toString();
+ String bucket = UUID.randomUUID().toString();
+ long containerID = writeDataAndGetContainer(true, volume, bucket);
+ // Overwrite the existing tree with a custom one for testing. We will check that it is returned properly from the
+ // API.
+ ContainerMerkleTreeWriter tree = buildTestTree(conf);
+ writeChecksumFileToDatanodes(containerID, tree);
+
+ // Verify trees match on all replicas.
+ // This test is expecting Ratis 3 data written on a 3 node cluster, so every node has a replica.
+ assertEquals(3, cluster.getHddsDatanodes().size());
+ List<DatanodeDetails> datanodeDetails = cluster.getHddsDatanodes().stream()
+ .map(HddsDatanodeService::getDatanodeDetails).collect(Collectors.toList());
+ for (DatanodeDetails dn: datanodeDetails) {
+ ContainerProtos.ContainerChecksumInfo containerChecksumInfo =
+ dnClient.getContainerChecksumInfo(containerID, dn);
+ assertTreesSortedAndMatch(tree.toProto(), containerChecksumInfo.getContainerMerkleTree());
+ }
+ }
+
+ @Test
+ public void testContainerChecksumWithBlockMissing() throws Exception {
+ // 1. Write data to a container.
+ // Read the key back and check its hash.
+ String volume = UUID.randomUUID().toString();
+ String bucket = UUID.randomUUID().toString();
+ Pair<Long, byte[]> containerAndData = getDataAndContainer(true, 20 * 1024 * 1024, volume, bucket);
+ long containerID = containerAndData.getLeft();
+ byte[] data = containerAndData.getRight();
+ // Get the datanodes where the container replicas are stored.
+ List<DatanodeDetails> dataNodeDetails = cluster.getStorageContainerManager().getContainerManager()
+ .getContainerReplicas(ContainerID.valueOf(containerID))
+ .stream().map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList());
+ assertEquals(3, dataNodeDetails.size());
+ HddsDatanodeService hddsDatanodeService = cluster.getHddsDatanode(dataNodeDetails.get(0));
+ DatanodeStateMachine datanodeStateMachine = hddsDatanodeService.getDatanodeStateMachine();
+ Container<?> container = datanodeStateMachine.getContainer().getContainerSet().getContainer(containerID);
+ KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
+ ContainerProtos.ContainerChecksumInfo oldContainerChecksumInfo = readChecksumFile(container.getContainerData());
+ KeyValueHandler kvHandler = (KeyValueHandler) datanodeStateMachine.getContainer().getDispatcher()
+ .getHandler(ContainerProtos.ContainerType.KeyValueContainer);
+
+ BlockManager blockManager = kvHandler.getBlockManager();
+ List<BlockData> blockDataList = blockManager.listBlock(container, -1, 100);
+ String chunksPath = container.getContainerData().getChunksPath();
+ long oldDataChecksum = oldContainerChecksumInfo.getContainerMerkleTree().getDataChecksum();
+
+ // 2. Delete some blocks to simulate missing blocks.
+ try (DBHandle db = BlockUtils.getDB(containerData, conf);
+ BatchOperation op = db.getStore().getBatchHandler().initBatchOperation()) {
+ for (int i = 0; i < blockDataList.size(); i += 2) {
+ BlockData blockData = blockDataList.get(i);
+ // Delete the block metadata from the container db
+ db.getStore().getBlockDataTable().deleteWithBatch(op, containerData.getBlockKey(blockData.getLocalID()));
+ // Delete the block file.
+ Files.deleteIfExists(Paths.get(chunksPath + "/" + blockData.getBlockID().getLocalID() + ".block"));
+ }
+ db.getStore().getBatchHandler().commitBatchOperation(op);
+ db.getStore().flushDB();
+ }
+
+ // TODO: Use On-demand container scanner to build the new container merkle tree. (HDDS-10374)
+ Files.deleteIfExists(getContainerChecksumFile(container.getContainerData()).toPath());
+ kvHandler.createContainerMerkleTree(container);
+ ContainerProtos.ContainerChecksumInfo containerChecksumAfterBlockDelete =
+ readChecksumFile(container.getContainerData());
+ long dataChecksumAfterBlockDelete = containerChecksumAfterBlockDelete.getContainerMerkleTree().getDataChecksum();
+ // Checksum should have changed after block delete.
+ assertNotEquals(oldDataChecksum, dataChecksumAfterBlockDelete);
+
+ // Since the container is already closed, we have manually updated the container checksum file.
+ // This doesn't update the checksum reported to SCM, and we need to trigger an ICR.
+ // Marking a container unhealthy will send an ICR.
+ kvHandler.markContainerUnhealthy(container, MetadataScanResult.deleted());
+ waitForDataChecksumsAtSCM(containerID, 2);
+
+ // 3. Reconcile the container.
+ cluster.getStorageContainerLocationClient().reconcileContainer(containerID);
+ // Compare and check if dataChecksum is same on all replicas.
+ waitForDataChecksumsAtSCM(containerID, 1);
+ ContainerProtos.ContainerChecksumInfo newContainerChecksumInfo = readChecksumFile(container.getContainerData());
+ assertTreesSortedAndMatch(oldContainerChecksumInfo.getContainerMerkleTree(),
+ newContainerChecksumInfo.getContainerMerkleTree());
+ TestHelper.validateData(KEY_NAME, data, store, volume, bucket);
+ }
+
+ @Test
+ public void testContainerChecksumChunkCorruption() throws Exception {
+ // 1. Write data to a container.
+ String volume = UUID.randomUUID().toString();
+ String bucket = UUID.randomUUID().toString();
+ Pair<Long, byte[]> containerAndData = getDataAndContainer(true, 20 * 1024 * 1024, volume, bucket);
+ long containerID = containerAndData.getLeft();
+ byte[] data = containerAndData.getRight();
+ // Get the datanodes where the container replicas are stored.
+ List<DatanodeDetails> dataNodeDetails = cluster.getStorageContainerManager().getContainerManager()
+ .getContainerReplicas(ContainerID.valueOf(containerID))
+ .stream().map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList());
+ assertEquals(3, dataNodeDetails.size());
+ HddsDatanodeService hddsDatanodeService = cluster.getHddsDatanode(dataNodeDetails.get(0));
+ DatanodeStateMachine datanodeStateMachine = hddsDatanodeService.getDatanodeStateMachine();
+ Container<?> container = datanodeStateMachine.getContainer().getContainerSet().getContainer(containerID);
+ KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
+ ContainerProtos.ContainerChecksumInfo oldContainerChecksumInfo = readChecksumFile(container.getContainerData());
+ KeyValueHandler kvHandler = (KeyValueHandler) datanodeStateMachine.getContainer().getDispatcher()
+ .getHandler(ContainerProtos.ContainerType.KeyValueContainer);
+
+ BlockManager blockManager = kvHandler.getBlockManager();
+ List<BlockData> blockDatas = blockManager.listBlock(container, -1, 100);
+ long oldDataChecksum = oldContainerChecksumInfo.getContainerMerkleTree().getDataChecksum();
+
+ // 2. Corrupt first chunk for all the blocks
+ try (DBHandle db = BlockUtils.getDB(containerData, conf);
+ BatchOperation op = db.getStore().getBatchHandler().initBatchOperation()) {
+ for (BlockData blockData : blockDatas) {
+ // Modify the block metadata to simulate chunk corruption.
+ ContainerProtos.BlockData.Builder blockDataBuilder = blockData.getProtoBufMessage().toBuilder();
+ blockDataBuilder.clearChunks();
+
+ ContainerProtos.ChunkInfo chunkInfo = blockData.getChunks().get(0);
+ ContainerProtos.ChecksumData.Builder checksumDataBuilder = ContainerProtos.ChecksumData.newBuilder()
+ .setBytesPerChecksum(chunkInfo.getChecksumData().getBytesPerChecksum())
+ .setType(chunkInfo.getChecksumData().getType());
+
+ for (ByteString checksum : chunkInfo.getChecksumData().getChecksumsList()) {
+ byte[] checksumBytes = checksum.toByteArray();
+ // Modify the checksum bytes to simulate corruption.
+ checksumBytes[0] = (byte) (checksumBytes[0] - 1);
+ checksumDataBuilder.addChecksums(ByteString.copyFrom(checksumBytes)).build();
+ }
+ chunkInfo = chunkInfo.toBuilder().setChecksumData(checksumDataBuilder.build()).build();
+ blockDataBuilder.addChunks(chunkInfo);
+ for (int i = 1; i < blockData.getChunks().size(); i++) {
+ blockDataBuilder.addChunks(blockData.getChunks().get(i));
+ }
+
+ // Modify the block metadata from the container db to simulate chunk corruption.
+ db.getStore().getBlockDataTable().putWithBatch(op, containerData.getBlockKey(blockData.getLocalID()),
+ BlockData.getFromProtoBuf(blockDataBuilder.build()));
+ }
+ db.getStore().getBatchHandler().commitBatchOperation(op);
+ db.getStore().flushDB();
+ }
+
+ Files.deleteIfExists(getContainerChecksumFile(container.getContainerData()).toPath());
+ kvHandler.createContainerMerkleTree(container);
+ // To set unhealthy for chunks that are corrupted.
+ ContainerProtos.ContainerChecksumInfo containerChecksumAfterChunkCorruption =
+ readChecksumFile(container.getContainerData());
+ long dataChecksumAfterAfterChunkCorruption = containerChecksumAfterChunkCorruption
+ .getContainerMerkleTree().getDataChecksum();
+ // Checksum should have changed after chunk corruption.
+ assertNotEquals(oldDataChecksum, dataChecksumAfterAfterChunkCorruption);
+
+ // 3. Set Unhealthy for first chunk of all blocks. This should be done by the scanner, Until then this is a
+ // manual step.
+ // TODO: Use On-demand container scanner to build the new container merkle tree (HDDS-10374)
+ Random random = new Random();
+ ContainerProtos.ContainerChecksumInfo.Builder builder = containerChecksumAfterChunkCorruption.toBuilder();
+ List<ContainerProtos.BlockMerkleTree> blockMerkleTreeList = builder.getContainerMerkleTree()
+ .getBlockMerkleTreeList();
+ builder.getContainerMerkleTreeBuilder().clearBlockMerkleTree();
+ for (ContainerProtos.BlockMerkleTree blockMerkleTree : blockMerkleTreeList) {
+ ContainerProtos.BlockMerkleTree.Builder blockMerkleTreeBuilder = blockMerkleTree.toBuilder();
+ List<ContainerProtos.ChunkMerkleTree.Builder> chunkMerkleTreeBuilderList =
+ blockMerkleTreeBuilder.getChunkMerkleTreeBuilderList();
+ chunkMerkleTreeBuilderList.get(0).setIsHealthy(false).setDataChecksum(random.nextLong());
+ blockMerkleTreeBuilder.setDataChecksum(random.nextLong());
+ builder.getContainerMerkleTreeBuilder().addBlockMerkleTree(blockMerkleTreeBuilder.build());
+ }
+ builder.getContainerMerkleTreeBuilder().setDataChecksum(random.nextLong());
+ Files.deleteIfExists(getContainerChecksumFile(container.getContainerData()).toPath());
+ writeContainerDataTreeProto(container.getContainerData(), builder.getContainerMerkleTree());
+
+ // Since the container is already closed, we have manually updated the container checksum file.
+ // This doesn't update the checksum reported to SCM, and we need to trigger an ICR.
+ // Marking a container unhealthy will send an ICR.
+ kvHandler.markContainerUnhealthy(container, MetadataScanResult.deleted());
+ waitForDataChecksumsAtSCM(containerID, 2);
+
+ // 4. Reconcile the container.
+ cluster.getStorageContainerLocationClient().reconcileContainer(containerID);
+ // Compare and check if dataChecksum is same on all replicas.
+ waitForDataChecksumsAtSCM(containerID, 1);
+ ContainerProtos.ContainerChecksumInfo newContainerChecksumInfo = readChecksumFile(container.getContainerData());
+ assertTreesSortedAndMatch(oldContainerChecksumInfo.getContainerMerkleTree(),
+ newContainerChecksumInfo.getContainerMerkleTree());
+ assertEquals(oldDataChecksum, newContainerChecksumInfo.getContainerMerkleTree().getDataChecksum());
+ TestHelper.validateData(KEY_NAME, data, store, volume, bucket);
+ }
+
+ @Test
+ public void testDataChecksumReportedAtSCM() throws Exception {
+ // 1. Write data to a container.
+ // Read the key back and check its hash.
+ String volume = UUID.randomUUID().toString();
+ String bucket = UUID.randomUUID().toString();
+ Pair<Long, byte[]> containerAndData = getDataAndContainer(true, 20 * 1024 * 1024, volume, bucket);
+ long containerID = containerAndData.getLeft();
+ byte[] data = containerAndData.getRight();
+ // Get the datanodes where the container replicas are stored.
+ List<DatanodeDetails> dataNodeDetails = cluster.getStorageContainerManager().getContainerManager()
+ .getContainerReplicas(ContainerID.valueOf(containerID))
+ .stream().map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList());
+ assertEquals(3, dataNodeDetails.size());
+ HddsDatanodeService hddsDatanodeService = cluster.getHddsDatanode(dataNodeDetails.get(0));
+ DatanodeStateMachine datanodeStateMachine = hddsDatanodeService.getDatanodeStateMachine();
+ Container<?> container = datanodeStateMachine.getContainer().getContainerSet().getContainer(containerID);
+ KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
+ ContainerProtos.ContainerChecksumInfo oldContainerChecksumInfo = readChecksumFile(container.getContainerData());
+ KeyValueHandler kvHandler = (KeyValueHandler) datanodeStateMachine.getContainer().getDispatcher()
+ .getHandler(ContainerProtos.ContainerType.KeyValueContainer);
+
+ long oldDataChecksum = oldContainerChecksumInfo.getContainerMerkleTree().getDataChecksum();
+ // Check non-zero checksum after container close
+ StorageContainerLocationProtocolClientSideTranslatorPB scmClient = cluster.getStorageContainerLocationClient();
+ List<HddsProtos.SCMContainerReplicaProto> containerReplicas = scmClient.getContainerReplicas(containerID,
+ ClientVersion.CURRENT_VERSION);
+ assertEquals(3, containerReplicas.size());
+ for (HddsProtos.SCMContainerReplicaProto containerReplica: containerReplicas) {
+ assertNotEquals(0, containerReplica.getDataChecksum());
+ }
+
+ // 2. Delete some blocks to simulate missing blocks.
+ BlockManager blockManager = kvHandler.getBlockManager();
+ List<BlockData> blockDataList = blockManager.listBlock(container, -1, 100);
+ String chunksPath = container.getContainerData().getChunksPath();
+ try (DBHandle db = BlockUtils.getDB(containerData, conf);
+ BatchOperation op = db.getStore().getBatchHandler().initBatchOperation()) {
+ for (int i = 0; i < blockDataList.size(); i += 2) {
+ BlockData blockData = blockDataList.get(i);
+ // Delete the block metadata from the container db
+ db.getStore().getBlockDataTable().deleteWithBatch(op, containerData.getBlockKey(blockData.getLocalID()));
+ // Delete the block file.
+ Files.deleteIfExists(Paths.get(chunksPath + "/" + blockData.getBlockID().getLocalID() + ".block"));
+ }
+ db.getStore().getBatchHandler().commitBatchOperation(op);
+ db.getStore().flushDB();
+ }
+
+ // TODO: Use On-demand container scanner to build the new container merkle tree. (HDDS-10374)
+ Files.deleteIfExists(getContainerChecksumFile(container.getContainerData()).toPath());
+ kvHandler.createContainerMerkleTree(container);
+ ContainerProtos.ContainerChecksumInfo containerChecksumAfterBlockDelete =
+ readChecksumFile(container.getContainerData());
+ long dataChecksumAfterBlockDelete = containerChecksumAfterBlockDelete.getContainerMerkleTree().getDataChecksum();
+ // Checksum should have changed after block delete.
+ assertNotEquals(oldDataChecksum, dataChecksumAfterBlockDelete);
+
+ // Since the container is already closed, we have manually updated the container checksum file.
+ // This doesn't update the checksum reported to SCM, and we need to trigger an ICR.
+ // Marking a container unhealthy will send an ICR.
+ kvHandler.markContainerUnhealthy(container, MetadataScanResult.deleted());
+ waitForDataChecksumsAtSCM(containerID, 2);
+ scmClient.reconcileContainer(containerID);
+
+ waitForDataChecksumsAtSCM(containerID, 1);
+ // Check non-zero checksum after container reconciliation
+ containerReplicas = scmClient.getContainerReplicas(containerID, ClientVersion.CURRENT_VERSION);
+ assertEquals(3, containerReplicas.size());
+ for (HddsProtos.SCMContainerReplicaProto containerReplica: containerReplicas) {
+ assertNotEquals(0, containerReplica.getDataChecksum());
+ }
+
+ // Check non-zero checksum after datanode restart
+ // Restarting all the nodes take more time in mini ozone cluster, so restarting only one node
+ cluster.restartHddsDatanode(0, true);
+ for (StorageContainerManager scm : cluster.getStorageContainerManagers()) {
+ cluster.restartStorageContainerManager(scm, false);
+ }
+ cluster.waitForClusterToBeReady();
+ waitForDataChecksumsAtSCM(containerID, 1);
+ containerReplicas = scmClient.getContainerReplicas(containerID, ClientVersion.CURRENT_VERSION);
+ assertEquals(3, containerReplicas.size());
+ for (HddsProtos.SCMContainerReplicaProto containerReplica: containerReplicas) {
+ assertNotEquals(0, containerReplica.getDataChecksum());
+ }
+ TestHelper.validateData(KEY_NAME, data, store, volume, bucket);
+ }
+
+ private void waitForDataChecksumsAtSCM(long containerID, int expectedSize) throws Exception {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ Set<Long> dataChecksums = cluster.getStorageContainerLocationClient().getContainerReplicas(containerID,
+ ClientVersion.CURRENT_VERSION).stream()
+ .map(HddsProtos.SCMContainerReplicaProto::getDataChecksum)
+ .collect(Collectors.toSet());
+ return dataChecksums.size() == expectedSize;
+ } catch (Exception ex) {
+ return false;
+ }
+ }, 500, 20000);
+ }
+
+ private Pair<Long, byte[]> getDataAndContainer(boolean close, int dataLen, String volumeName, String bucketName)
+ throws Exception {
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ byte[] data = randomAlphabetic(dataLen).getBytes(UTF_8);
+ // Write Key
+ try (OzoneOutputStream os = TestHelper.createKey(KEY_NAME, RATIS, THREE, dataLen, store, volumeName, bucketName)) {
+ IOUtils.write(data, os);
+ }
+
+ long containerID = bucket.getKey(KEY_NAME).getOzoneKeyLocations().stream()
+ .findFirst().get().getContainerID();
+ if (close) {
+ TestHelper.waitForContainerClose(cluster, containerID);
+ TestHelper.waitForScmContainerState(cluster, containerID, HddsProtos.LifeCycleState.CLOSED);
+ }
+ return Pair.of(containerID, data);
+ }
+
+ private long writeDataAndGetContainer(boolean close, String volume, String bucket) throws Exception {
+ return getDataAndContainer(close, 5, volume, bucket).getLeft();
+ }
+
+ public static void writeChecksumFileToDatanodes(long containerID, ContainerMerkleTreeWriter tree) throws Exception {
+ // Write Container Merkle Tree
+ for (HddsDatanodeService dn : cluster.getHddsDatanodes()) {
+ KeyValueHandler keyValueHandler =
+ (KeyValueHandler) dn.getDatanodeStateMachine().getContainer().getDispatcher()
+ .getHandler(ContainerProtos.ContainerType.KeyValueContainer);
+ KeyValueContainer keyValueContainer =
+ (KeyValueContainer) dn.getDatanodeStateMachine().getContainer().getController()
+ .getContainer(containerID);
+ if (keyValueContainer != null) {
+ keyValueHandler.getChecksumManager().writeContainerDataTree(
+ keyValueContainer.getContainerData(), tree);
+ }
+ }
+ }
+
+ private static void setSecretKeysConfig() {
+ // Secret key lifecycle configs.
+ conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "500s");
+ conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "500s");
+ conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "500s");
+
+ // enable tokens
+ conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true);
+ conf.setBoolean(HDDS_CONTAINER_TOKEN_ENABLED, true);
+ }
+
+ private static void createCredentialsInKDC() throws Exception {
+ ScmConfig scmConfig = conf.getObject(ScmConfig.class);
+ SCMHTTPServerConfig httpServerConfig =
+ conf.getObject(SCMHTTPServerConfig.class);
+ createPrincipal(ozoneKeytab, scmConfig.getKerberosPrincipal());
+ createPrincipal(spnegoKeytab, httpServerConfig.getKerberosPrincipal());
+ createPrincipal(testUserKeytab, testUserPrincipal);
+ }
+
+ private static void createPrincipal(File keytab, String... principal)
+ throws Exception {
+ miniKdc.createPrincipal(keytab, principal);
+ }
+
+ private static void startMiniKdc() throws Exception {
+ Properties securityProperties = MiniKdc.createConf();
+ miniKdc = new MiniKdc(securityProperties, workDir);
+ miniKdc.start();
+ }
+
+ private static void setSecureConfig() throws IOException {
+ conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+ host = InetAddress.getLocalHost().getCanonicalHostName()
+ .toLowerCase();
+ conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name());
+ String curUser = UserGroupInformation.getCurrentUser().getUserName();
+ conf.set(OZONE_ADMINISTRATORS, curUser);
+ String realm = miniKdc.getRealm();
+ String hostAndRealm = host + "@" + realm;
+ conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
+ conf.set(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + hostAndRealm);
+ conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
+ conf.set(OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_OM/" + hostAndRealm);
- conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
++ conf.set(HDDS_DATANODE_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm);
+
+ ozoneKeytab = new File(workDir, "scm.keytab");
+ spnegoKeytab = new File(workDir, "http.keytab");
+ testUserKeytab = new File(workDir, "testuser.keytab");
+ testUserPrincipal = "test@" + realm;
+
+ conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, ozoneKeytab.getAbsolutePath());
+ conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY, spnegoKeytab.getAbsolutePath());
+ conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, ozoneKeytab.getAbsolutePath());
+ conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE, spnegoKeytab.getAbsolutePath());
- conf.set(DFS_DATANODE_KERBEROS_KEYTAB_FILE_KEY, ozoneKeytab.getAbsolutePath());
++ conf.set(HDDS_DATANODE_KERBEROS_KEYTAB_FILE_KEY, ozoneKeytab.getAbsolutePath());
+ }
+
+ private static void startCluster() throws Exception {
+ OzoneManager.setTestSecureOmFlag(true);
+ cluster = MiniOzoneCluster.newHABuilder(conf)
+ .setSCMServiceId("SecureSCM")
+ .setNumOfStorageContainerManagers(3)
+ .setNumOfOzoneManagers(1)
+ .build();
+ cluster.waitForClusterToBeReady();
+ rpcClient = OzoneClientFactory.getRpcClient(conf);
+ store = rpcClient.getObjectStore();
+ SecretKeyClient secretKeyClient = cluster.getStorageContainerManager().getSecretKeyManager();
+ CertificateClient certClient = cluster.getStorageContainerManager().getScmCertificateClient();
+ dnClient = new DNContainerOperationClient(conf, certClient, secretKeyClient);
+ }
+}
diff --cc hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
index 8545951f92,4e8cd20f49..56fc47132b
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
@@@ -565,46 -589,29 +594,39 @@@ public class ContainerHealthTask extend
unhealthyContainerStateStatsMap);
}
- if (container.isUnderReplicated()
- && !recordForStateExists.contains(
- UnHealthyContainerStates.UNDER_REPLICATED.toString())) {
- records.add(recordForState(
- container, UnHealthyContainerStates.UNDER_REPLICATED, time));
- populateContainerStats(container,
- UnHealthyContainerStates.UNDER_REPLICATED,
- unhealthyContainerStateStatsMap);
- }
-
- if (container.isOverReplicated()
- && !recordForStateExists.contains(
- UnHealthyContainerStates.OVER_REPLICATED.toString())) {
- records.add(recordForState(
- container, UnHealthyContainerStates.OVER_REPLICATED, time));
- populateContainerStats(container,
- UnHealthyContainerStates.OVER_REPLICATED,
- unhealthyContainerStateStatsMap);
+ if (container.isUnderReplicated()) {
+ boolean shouldAddRecord = !recordForStateExists.contains(UnHealthyContainerStates.UNDER_REPLICATED.toString());
+ if (shouldAddRecord) {
+ records.add(recordForState(container, UnHealthyContainerStates.UNDER_REPLICATED, time));
+ }
+ populateContainerStats(container, UnHealthyContainerStates.UNDER_REPLICATED, unhealthyContainerStateStatsMap);
}
- if (container.isMisReplicated()
- && !recordForStateExists.contains(
- UnHealthyContainerStates.MIS_REPLICATED.toString())) {
- records.add(recordForState(
- container, UnHealthyContainerStates.MIS_REPLICATED, time));
- populateContainerStats(container,
- UnHealthyContainerStates.MIS_REPLICATED,
- unhealthyContainerStateStatsMap);
+ if (container.isOverReplicated()) {
+ boolean shouldAddRecord = !recordForStateExists.contains(UnHealthyContainerStates.OVER_REPLICATED.toString());
+ if (shouldAddRecord) {
+ records.add(recordForState(container, UnHealthyContainerStates.OVER_REPLICATED, time));
+ }
+ populateContainerStats(container, UnHealthyContainerStates.OVER_REPLICATED, unhealthyContainerStateStatsMap);
}
+ if (container.isDataChecksumMismatched()
+ && !recordForStateExists.contains(
+ UnHealthyContainerStates.REPLICA_MISMATCH.toString())) {
+ records.add(recordForState(
+ container, UnHealthyContainerStates.REPLICA_MISMATCH, time));
+ populateContainerStats(container,
+ UnHealthyContainerStates.REPLICA_MISMATCH,
+ unhealthyContainerStateStatsMap);
+ }
+
+ if (container.isMisReplicated()) {
+ boolean shouldAddRecord = !recordForStateExists.contains(UnHealthyContainerStates.MIS_REPLICATED.toString());
+ if (shouldAddRecord) {
+ records.add(recordForState(container, UnHealthyContainerStates.MIS_REPLICATED, time));
+ }
+ populateContainerStats(container, UnHealthyContainerStates.MIS_REPLICATED, unhealthyContainerStateStatsMap);
+ }
return records;
}
diff --cc hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UnhealthyContainerReplicaMismatchAction.java
index 3ff8a51ccb,0000000000..7283858e80
mode 100644,000000..100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UnhealthyContainerReplicaMismatchAction.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UnhealthyContainerReplicaMismatchAction.java
@@@ -1,99 -1,0 +1,99 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.upgrade;
+
+import static org.apache.hadoop.ozone.recon.upgrade.ReconLayoutFeature.UNHEALTHY_CONTAINER_REPLICA_MISMATCH;
+import static org.apache.hadoop.ozone.recon.upgrade.ReconUpgradeAction.UpgradeActionType.FINALIZE;
- import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK;
- import static org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UNHEALTHY_CONTAINERS_TABLE_NAME;
++import static org.apache.ozone.recon.schema.ContainerSchemaDefinition.UNHEALTHY_CONTAINERS_TABLE_NAME;
++import static org.apache.ozone.recon.schema.SqlDbUtils.TABLE_EXISTS_CHECK;
+import static org.jooq.impl.DSL.field;
+import static org.jooq.impl.DSL.name;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Arrays;
+import javax.sql.DataSource;
+import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
- import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition;
++import org.apache.ozone.recon.schema.ContainerSchemaDefinition;
+import org.jooq.DSLContext;
+import org.jooq.impl.DSL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Upgrade action for handling the addition of a new unhealthy container state in Recon, which will be for containers,
+ * that have replicas with different data checksums.
+ */
+@UpgradeActionRecon(feature = UNHEALTHY_CONTAINER_REPLICA_MISMATCH, type = FINALIZE)
+public class UnhealthyContainerReplicaMismatchAction implements ReconUpgradeAction {
+ private static final Logger LOG = LoggerFactory.getLogger(InitialConstraintUpgradeAction.class);
+ private DataSource dataSource;
+ private DSLContext dslContext;
+
+ @Override
+ public void execute(ReconStorageContainerManagerFacade scmFacade) throws Exception {
+ this.dataSource = scmFacade.getDataSource();
+ try (Connection conn = dataSource.getConnection()) {
+ if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) {
+ return;
+ }
+ dslContext = DSL.using(conn);
+ // Drop the existing constraint
+ dropConstraint();
+ // Add the updated constraint with all enum states
+ addUpdatedConstraint();
+ } catch (SQLException e) {
+ throw new SQLException("Failed to execute UnhealthyContainerReplicaMismatchAction", e);
+ }
+ }
+
+ /**
+ * Drops the existing constraint from the UNHEALTHY_CONTAINERS table.
+ */
+ private void dropConstraint() {
+ String constraintName = UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1";
+ dslContext.alterTable(UNHEALTHY_CONTAINERS_TABLE_NAME)
+ .dropConstraint(constraintName)
+ .execute();
+ LOG.debug("Dropped the existing constraint: {}", constraintName);
+ }
+
+ /**
+ * Adds the updated constraint directly within this class.
+ */
+ private void addUpdatedConstraint() {
+ String[] enumStates = Arrays
+ .stream(ContainerSchemaDefinition.UnHealthyContainerStates.values())
+ .map(Enum::name)
+ .toArray(String[]::new);
+
+ dslContext.alterTable(ContainerSchemaDefinition.UNHEALTHY_CONTAINERS_TABLE_NAME)
+ .add(DSL.constraint(ContainerSchemaDefinition.UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1")
+ .check(field(name("container_state"))
+ .in(enumStates)))
+ .execute();
+
+ LOG.info("Added the updated constraint to the UNHEALTHY_CONTAINERS table for enum state values: {}",
+ Arrays.toString(enumStates));
+ }
+
+ @Override
+ public UpgradeActionType getType() {
+ return FINALIZE;
+ }
+}
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/datanodes/Datanodes.test.tsx
index a169e1ce34,4ba0486db3..0b9f709a32
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/datanodes/Datanodes.test.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/datanodes/Datanodes.test.tsx
@@@ -16,21 -16,21 +16,16 @@@
* limitations under the License.
*/
import React from 'react';
--import {
-- fireEvent,
-- render,
-- screen,
-- waitFor
--} from '@testing-library/react';
++import {fireEvent, render, screen, waitFor} from '@testing-library/react';
import userEvent from '@testing-library/user-event';
--import { rest } from "msw";
--import { vi } from 'vitest';
++import {rest} from "msw";
++import {vi} from 'vitest';
import Datanodes from '@/v2/pages/datanodes/datanodes';
import * as commonUtils from '@/utils/common';
- import { datanodeServer } from '@/__tests__/mocks/datanodeMocks/datanodeServer';
- import { datanodeLocators, searchInputLocator } from '@/__tests__/locators/locators';
- import { waitForDNTable } from '@/__tests__/utils/datanodes.utils';
-import { datanodeServer } from '@tests/mocks/datanodeMocks/datanodeServer';
-import { datanodeLocators, searchInputLocator } from '@tests//locators/locators';
-import { waitForDNTable } from '@tests/utils/datanodes.utils';
++import {datanodeServer} from '@tests/mocks/datanodeMocks/datanodeServer';
++import {datanodeLocators, searchInputLocator} from '@tests//locators/locators';
++import {waitForDNTable} from '@tests/utils/datanodes.utils';
// Mock utility functions
vi.spyOn(commonUtils, 'showDataFetchError');
@@@ -188,4 -188,4 +183,4 @@@ describe('Datanodes Component', () =>
expect(commonUtils.showDataFetchError).toHaveBeenCalledWith('AxiosError: Request failed with status code 500')
);
});
--});
++});
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/datanodes/DatanodesTable.test.tsx
index f1be5362ec,555ab850e8..06b878055c
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/datanodes/DatanodesTable.test.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/datanodes/DatanodesTable.test.tsx
@@@ -17,18 -17,18 +17,13 @@@
*/
import React from 'react';
--import { vi } from 'vitest';
--import {
-- fireEvent,
-- render,
-- screen,
-- waitFor
--} from '@testing-library/react';
++import {vi} from 'vitest';
++import {fireEvent, render, screen} from '@testing-library/react';
--import { DatanodeTableProps } from '@/v2/types/datanode.types';
++import {DatanodeTableProps} from '@/v2/types/datanode.types';
import DatanodesTable from '@/v2/components/tables/datanodesTable';
- import { datanodeServer } from '@/__tests__/mocks/datanodeMocks/datanodeServer';
- import { waitForDNTable } from '@/__tests__/utils/datanodes.utils';
-import { datanodeServer } from '@tests/mocks/datanodeMocks/datanodeServer';
-import { waitForDNTable } from '@tests/utils/datanodes.utils';
++import {datanodeServer} from '@tests/mocks/datanodeMocks/datanodeServer';
++import {waitForDNTable} from '@tests/utils/datanodes.utils';
const defaultProps: DatanodeTableProps = {
loading: false,
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/mocks/pipelineMocks/pipelinesServer.ts
index 0000000000,1b2c48484e..bb2f40a70d
mode 000000,100644..100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/mocks/pipelineMocks/pipelinesServer.ts
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/mocks/pipelineMocks/pipelinesServer.ts
@@@ -1,0 -1,35 +1,35 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
-import { setupServer } from "msw/node";
-import { rest } from "msw";
++import {setupServer} from "msw/node";
++import {rest} from "msw";
+
+ import * as mockResponses from "./pipelineResponseMocks";
+
+ const handlers = [
+ rest.get("api/v1/pipelines", (req, res, ctx) => {
+ return res(
+ ctx.status(200),
+ ctx.json(mockResponses.PipelinesResponse)
+ );
+ })
+ ];
+
+
+ //This will configure a request mocking server using MSW
+ export const pipelineServer = setupServer(...handlers);
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/pipelines/Pipelines.test.tsx
index 0000000000,f703ec4102..57fc2cc2f4
mode 000000,100644..100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/pipelines/Pipelines.test.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/pipelines/Pipelines.test.tsx
@@@ -1,0 -1,142 +1,137 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ import React from 'react';
-import {
- render,
- screen,
- waitFor,
- fireEvent
-} from '@testing-library/react';
-import { rest } from 'msw';
-import { vi } from 'vitest';
++import {fireEvent, render, screen, waitFor} from '@testing-library/react';
++import {rest} from 'msw';
++import {vi} from 'vitest';
+
+ import Pipelines from '@/v2/pages/pipelines/pipelines';
+ import * as commonUtils from '@/utils/common';
-import { pipelineServer } from '@tests/mocks/pipelineMocks/pipelinesServer';
-import { pipelineLocators, searchInputLocator } from '@tests/locators/locators';
-import { waitForPipelineTable } from '@tests/utils/pipelines.utils';
++import {pipelineServer} from '@tests/mocks/pipelineMocks/pipelinesServer';
++import {pipelineLocators, searchInputLocator} from '@tests/locators/locators';
++import {waitForPipelineTable} from '@tests/utils/pipelines.utils';
+
+ // Mock utility functions
+ vi.spyOn(commonUtils, 'showDataFetchError');
+
+ vi.mock('@/components/autoReloadPanel/autoReloadPanel', () => ({
+ default: () => <div data-testid="auto-reload-panel" />,
+ }));
+ vi.mock('@/v2/components/select/multiSelect.tsx', () => ({
+ default: ({ onChange }: { onChange: Function }) => (
+ <select data-testid="multi-select" onChange={(e) => onChange(e.target.value)}>
+ <option value="pipelineId">Pipeline ID</option>
+ <option value="replicationType">Replication Type & Factor</option>
+ <option value="status">Status</option>
+ </select>
+ ),
+ }));
+
+ describe('Pipelines Component', () => {
+ // Start and stop MSW server before and after all tests
+ beforeAll(() => pipelineServer.listen());
+ afterEach(async () => pipelineServer.resetHandlers());
+ afterAll(() => pipelineServer.close());
+
+ test('renders component correctly', () => {
+ render(<Pipelines />);
+
+ // Verify core components are rendered
+ expect(screen.getByText(/Pipelines/)).toBeInTheDocument();
+ expect(screen.getByTestId('auto-reload-panel')).toBeInTheDocument();
+ expect(screen.getByTestId('multi-select')).toBeInTheDocument();
+ expect(screen.getByTestId(searchInputLocator)).toBeInTheDocument();
+ });
+
+ test('Loads data on mount', async () => {
+ render(<Pipelines />);
+
+ // Wait for the data to load into the table
+ const table = await waitForPipelineTable();
+ expect(table).toBeInTheDocument();
+
+ // Verify data is displayed, from pipelineMocks
+ expect(table).toHaveTextContent('pipeline-1');
+ expect(table).toHaveTextContent('CLOSED');
+ });
+
+ test('Renders table with correct number of rows', async () => {
+ render(<Pipelines />);
+
+ // Wait for the data to load into the table
+ const rows = await waitFor(() => screen.getAllByTestId(pipelineLocators.pipelineRowRegex));
+ expect(rows).toHaveLength(3); // Based on the mocked Pipeline response
+ });
+
+ test('Displays no data message if the pipelines API returns an empty array', async () => {
+ pipelineServer.use(
+ rest.get('api/v1/pipelines', (req, res, ctx) => {
+ return res(ctx.status(200), ctx.json({ totalCount: 0, pipelines: [] }));
+ })
+ );
+
+ render(<Pipelines />);
+
+ // Wait for the no data message
+ await waitFor(() => expect(screen.getByText('No Data')).toBeInTheDocument());
+ });
+
+ test('Handles search input change', async () => {
+ render(<Pipelines />);
+ await waitForPipelineTable();
+
+ const searchInput = screen.getByTestId(searchInputLocator);
+ fireEvent.change(searchInput, {
+ target: { value: 'pipeline-1' }
+ });
+ // Sleep for 310ms to allow debounced search to take effect
+ await new Promise((r) => { setTimeout(r, 310) });
+ const rows = await waitFor(() => screen.getAllByTestId(pipelineLocators.pipelineRowRegex));
+ await waitFor(() => expect(rows).toHaveLength(1));
+ });
+
+ test('Displays a message when no results match the search term', async () => {
+ render(<Pipelines />);
+ const searchInput = screen.getByTestId(searchInputLocator);
+
+ // Type a term that doesn't match any datanode
+ fireEvent.change(searchInput, {
+ target: { value: 'nonexistent-pipeline' }
+ });
+
+ // Verify that no results message is displayed
+ await waitFor(() => expect(screen.getByText('No Data')).toBeInTheDocument());
+ });
+
+ test('Handles API errors gracefully by showing error message', async () => {
+ // Set up MSW to return an error for the datanode API
+ pipelineServer.use(
+ rest.get('api/v1/pipelines', (req, res, ctx) => {
+ return res(ctx.status(500), ctx.json({ error: 'Internal Server Error' }));
+ })
+ );
+
+ render(<Pipelines />);
+
+ // Wait for the error to be handled
+ await waitFor(() =>
+ expect(commonUtils.showDataFetchError).toHaveBeenCalledWith('AxiosError: Request failed with status code 500')
+ );
+ });
-});
++});
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/pipelines/PipelinesTable.test.tsx
index 0000000000,3cd72a2093..50e4486ff6
mode 000000,100644..100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/pipelines/PipelinesTable.test.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/pipelines/PipelinesTable.test.tsx
@@@ -1,0 -1,186 +1,186 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ import React from 'react';
-import { render, screen, fireEvent, waitFor } from '@testing-library/react';
++import {fireEvent, render, screen, waitFor} from '@testing-library/react';
+ import PipelinesTable from '@/v2/components/tables/pipelinesTable';
-import { Pipeline, PipelinesTableProps } from '@/v2/types/pipelines.types';
-import { pipelineLocators } from '@tests/locators/locators';
-import { userEvent } from '@testing-library/user-event';
++import {Pipeline, PipelinesTableProps} from '@/v2/types/pipelines.types';
++import {pipelineLocators} from '@tests/locators/locators';
++import {userEvent} from '@testing-library/user-event';
+
+ // Mock to disable scroll behaviour
+ // We are running test inside virtual DOM, so scrolling isn't available as a normal browser
+ // AntD tries to scroll to the top after changing pages. This disables the behaviour.
+ vi.mock('antd/es/_util/scrollTo', () => ({
+ default: vi.fn()
+ }));
+
+
+ const defaultProps: PipelinesTableProps = {
+ loading: false,
+ data: [],
+ selectedColumns: [
+ { label: 'Pipeline ID', value: 'pipelineId' },
+ { label: 'Status', value: 'status' },
+ ],
+ searchTerm: '',
+ };
+
+ function getPipelineWith(
+ id: string,
+ status: 'OPEN' | 'CLOSING' | 'QUASI_CLOSED' | 'CLOSED' | 'UNHEALTHY' | 'INVALID' | 'DELETED' | 'DORMANT',
+ replicationFactor: string
+ ): Pipeline {
+ return {
+ pipelineId: id,
+ status,
+ containers: 10,
+ datanodes: [],
+ leaderNode: 'node-1',
+ replicationFactor,
+ replicationType: 'RATIS',
+ duration: 10000,
+ leaderElections: 1,
+ lastLeaderElection: 1000,
+ };
+ }
+
+ describe('PipelinesTable Component', () => {
+ test('Renders table with data', async () => {
+ render(
+ <PipelinesTable
+ {...defaultProps}
+ data={[getPipelineWith('pipeline-1', 'UNHEALTHY', 'THREE')]}
+ />
+ );
+
+ // Verify table is rendered
+ await waitFor(() => screen.getByText('pipeline-1'));
+ expect(screen.getByText('UNHEALTHY')).toBeInTheDocument();
+ });
+
+ test('Filters data based on search term', async () => {
+ render(
+ <PipelinesTable
+ {...defaultProps}
+ searchTerm="pipeline-1"
+ data={[
+ getPipelineWith('pipeline-1', 'OPEN', 'THREE'),
+ getPipelineWith('pipeline-2', 'QUASI_CLOSED', 'ONE'),
+ ]}
+ />
+ );
+
+ // Verify only matching rows are displayed
+ expect(screen.getByText('pipeline-1')).toBeInTheDocument();
+ expect(screen.queryByText('pipeline-2')).not.toBeInTheDocument();
+ });
+
+ test('Renders correct columns based on selectedColumns', async () => {
+ render(
+ <PipelinesTable
+ {...defaultProps}
+ selectedColumns={[
+ { label: 'Pipeline ID', value: 'pipelineId' },
+ ]}
+ data={[getPipelineWith('pipeline-1', 'CLOSED', 'ONE')]}
+ />
+ );
+
+ // Verify only the selected columns are rendered
+ expect(screen.getByText('Pipeline ID')).toBeInTheDocument();
+ expect(screen.queryByText('Replication Type & Factor')).not.toBeInTheDocument();
+ expect(screen.queryByText('Status')).not.toBeInTheDocument();
+ });
+
+ test('Sorts table rows by pipeline ID', async () => {
+ render(
+ <PipelinesTable
+ {...defaultProps}
+ data={[
+ getPipelineWith('pipeline-2', 'CLOSED', 'ONE'),
+ getPipelineWith('pipeline-1', 'DELETED', 'THREE'),
+ ]}
+ />
+ );
+
+ // Simulate sorting by pipeline ID
+ const pipelineIdHeader = screen.getByText('Pipeline ID');
+ fireEvent.click(pipelineIdHeader);
+
+ // Verify rows are sorted
+ const rows = screen.getAllByTestId(pipelineLocators.pipelineRowRegex);
+ expect(rows[0]).toHaveTextContent('pipeline-1');
+ expect(rows[1]).toHaveTextContent('pipeline-2');
+ });
+
+ test('Sorts table rows by pipeline status', async () => {
+ render(
+ <PipelinesTable
+ {...defaultProps}
+ data={[
+ getPipelineWith('pipeline-2', 'CLOSED', 'ONE'),
+ getPipelineWith('pipeline-1', 'DELETED', 'THREE'),
+ ]}
+ />
+ );
+
+ // Simulate sorting by pipeline ID
+ const pipelineIdHeader = screen.getByText('Status');
+ fireEvent.click(pipelineIdHeader);
+
+ // Verify rows are sorted
+ const rows = screen.getAllByTestId(pipelineLocators.pipelineRowRegex);
+ expect(rows[0]).toHaveTextContent('CLOSED');
+ expect(rows[1]).toHaveTextContent('DELETED');
+ });
+
+ test('Pagination works properly', async () => {
+ render(
+ <PipelinesTable
+ {...defaultProps}
+ data={[
+ getPipelineWith('pipeline-1', 'DELETED', 'THREE'),
+ getPipelineWith('pipeline-2', 'CLOSED', 'ONE'),
+ getPipelineWith('pipeline-3', 'OPEN', 'THREE'),
+ getPipelineWith('pipeline-4', 'OPEN', 'ONE'),
+ getPipelineWith('pipeline-5', 'OPEN', 'ONE'),
+ getPipelineWith('pipeline-6', 'OPEN', 'ONE'),
+ getPipelineWith('pipeline-7', 'OPEN', 'ONE'),
+ getPipelineWith('pipeline-8', 'OPEN', 'ONE'),
+ getPipelineWith('pipeline-9', 'OPEN', 'ONE'),
+ getPipelineWith('pipeline-10', 'OPEN', 'ONE'),
+ getPipelineWith('pipeline-11', 'OPEN', 'THREE')
+ ]}
+ />
+ );
+
+ try {
+ // Simulate clicking the next page button
+ const nextPageBtn = screen.getByTitle('Next Page');
+ userEvent.click(nextPageBtn, { pointerEventsCheck: 0 });
+ // Verify rows are sorted
+ const rows = screen.getAllByTestId(pipelineLocators.pipelineRowRegex);
+ expect(rows[0]).toHaveTextContent('pipeline-11');
+ } catch(err) {
+ if (err instanceof ReferenceError) {
+ console.log("Encountered error: ", err);
+ }
+ }
+ });
+ });
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/utils/pipelines.utils.tsx
index 0000000000,9042ef6dce..40a89859e1
mode 000000,100644..100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/utils/pipelines.utils.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/utils/pipelines.utils.tsx
@@@ -1,0 -1,23 +1,23 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
-import { waitFor, screen } from "@testing-library/react";
++import {screen, waitFor} from "@testing-library/react";
+
+ export const waitForPipelineTable = async () => {
+ return waitFor(() => screen.getByTestId('pipelines-table'));
-}
++}
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/duMetadata/duMetadata.tsx
index b5dc002c1f,609b768aec..6963675b4c
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/duMetadata/duMetadata.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/duMetadata/duMetadata.tsx
@@@ -16,15 -16,15 +16,15 @@@
* limitations under the License.
*/
--import React, { useRef, useState } from 'react';
++import React, {useRef, useState} from 'react';
import moment from 'moment';
--import axios, { AxiosError } from 'axios';
--import { Table } from 'antd';
++import axios, {AxiosError} from 'axios';
++import {Table} from 'antd';
--import { AxiosGetHelper, cancelRequests, PromiseAllSettledGetHelper } from '@/utils/axiosRequestHelper';
--import { byteToSize, checkResponseError, removeDuplicatesAndMerge, showDataFetchError } from '@/utils/common';
++import {AxiosGetHelper, cancelRequests, PromiseAllSettledGetHelper} from '@/utils/axiosRequestHelper';
++import {byteToSize, checkResponseError, removeDuplicatesAndMerge, showDataFetchError} from '@/utils/common';
--import { Acl } from '@/v2/types/acl.types';
++import {Acl} from '@/v2/types/acl.types';
// ------------- Types -------------- //
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/tables/containersTable.tsx
index 1bb1b5456b,b8520c11ee..a68c9928ee
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/tables/containersTable.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/tables/containersTable.tsx
@@@ -16,23 -16,23 +16,23 @@@
* limitations under the License.
*/
--import React, { useRef } from 'react';
++import React, {useRef} from 'react';
import filesize from 'filesize';
--import { AxiosError } from 'axios';
--import { Popover, Table } from 'antd';
--import {
-- ColumnsType,
-- TablePaginationConfig
--} from 'antd/es/table';
--import { NodeIndexOutlined } from '@ant-design/icons';
++import {AxiosError} from 'axios';
++import {Popover, Table} from 'antd';
++import {ColumnsType, TablePaginationConfig} from 'antd/es/table';
++import {NodeIndexOutlined} from '@ant-design/icons';
--import { getFormattedTime } from '@/v2/utils/momentUtils';
--import { showDataFetchError } from '@/utils/common';
--import { AxiosGetHelper } from '@/utils/axiosRequestHelper';
++import {getFormattedTime} from '@/v2/utils/momentUtils';
++import {showDataFetchError} from '@/utils/common';
++import {AxiosGetHelper} from '@/utils/axiosRequestHelper';
import {
-- Container, ContainerKeysResponse, ContainerReplica,
++ Container,
++ ContainerKeysResponse,
++ ContainerReplica,
ContainerTableProps,
-- ExpandedRowState, KeyResponse
++ ExpandedRowState,
++ KeyResponse
} from '@/v2/types/container.types';
const size = filesize.partial({ standard: 'iec' });
@@@ -256,4 -256,4 +256,4 @@@ const ContainerTable: React.FC<Containe
);
}
--export default ContainerTable;
++export default ContainerTable;
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/tables/pipelinesTable.tsx
index 6c07749436,e5a3cf8195..9d0a8704a1
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/tables/pipelinesTable.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/components/tables/pipelinesTable.tsx
@@@ -16,18 -16,18 +16,15 @@@
* limitations under the License.
*/
- import React from 'react';
-import React, { HTMLAttributes } from 'react';
++import React, {HTMLAttributes} from 'react';
--import Table, {
-- ColumnsType,
-- TablePaginationConfig
--} from 'antd/es/table';
++import Table, {ColumnsType, TablePaginationConfig} from 'antd/es/table';
import Tooltip from 'antd/es/tooltip';
--import { InfoCircleOutlined } from '@ant-design/icons';
++import {InfoCircleOutlined} from '@ant-design/icons';
--import { ReplicationIcon } from '@/utils/themeIcons';
--import { getDurationFromTimestamp, getTimeDiffFromTimestamp } from '@/v2/utils/momentUtils';
--import { Pipeline, PipelinesTableProps, PipelineStatusList } from '@/v2/types/pipelines.types';
++import {ReplicationIcon} from '@/utils/themeIcons';
++import {getDurationFromTimestamp, getTimeDiffFromTimestamp} from '@/v2/utils/momentUtils';
++import {Pipeline, PipelinesTableProps, PipelineStatusList} from '@/v2/types/pipelines.types';
// TODO: When Datanodes PR gets merged remove these declarations
@@@ -208,4 -212,4 +209,4 @@@ const PipelinesTable: React.FC<Pipeline
)
}
--export default PipelinesTable;
++export default PipelinesTable;
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx
index 43cc7f69e3,19f4972c66..b96ddff557
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx
@@@ -18,27 -18,27 +18,14 @@@
import React from 'react';
import moment from 'moment';
--import {
-- Row,
-- Button,
-- Input,
-- Menu,
-- Dropdown,
-- Tooltip
--} from 'antd';
--import { MenuProps } from 'antd/es/menu';
--import {
-- CloseOutlined,
-- InfoCircleOutlined,
-- LeftOutlined,
-- LoadingOutlined,
-- RedoOutlined
--} from '@ant-design/icons';
--
--import { DetailPanel } from '@/components/rightDrawer/rightDrawer';
--import { EChart } from '@/components/eChart/eChart';
--import { byteToSize, showDataFetchError } from '@/utils/common';
--import { AxiosGetHelper, cancelRequests } from '@/utils/axiosRequestHelper';
++import {Button, Dropdown, Input, Menu, Row, Tooltip} from 'antd';
++import {MenuProps} from 'antd/es/menu';
++import {InfoCircleOutlined, LeftOutlined, LoadingOutlined, RedoOutlined} from '@ant-design/icons';
++
++import {DetailPanel} from '@/components/rightDrawer/rightDrawer';
++import {EChart} from '@/components/eChart/eChart';
++import {byteToSize, showDataFetchError} from '@/utils/common';
++import {AxiosGetHelper, cancelRequests} from '@/utils/axiosRequestHelper';
import './diskUsage.less';
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/insights/om/om.tsx
index f092708348,06455aef47..fd8d7e2da9
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/insights/om/om.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/insights/om/om.tsx
@@@ -19,17 -19,17 +19,17 @@@
import React from 'react';
import moment from 'moment';
import filesize from 'filesize';
--import { Table, Tabs, Menu, Dropdown, Tooltip } from 'antd';
--import { MenuProps } from 'antd/es/menu';
--import { TablePaginationConfig } from 'antd/es/table';
--import { FunnelPlotFilled, InfoCircleOutlined } from '@ant-design/icons';
--import { ActionMeta, ValueType } from "react-select";
++import {Dropdown, Menu, Table, Tabs, Tooltip} from 'antd';
++import {MenuProps} from 'antd/es/menu';
++import {TablePaginationConfig} from 'antd/es/table';
++import {FunnelPlotFilled, InfoCircleOutlined} from '@ant-design/icons';
++import {ActionMeta, ValueType} from "react-select";
import CreatableSelect from "react-select/creatable";
--import { ColumnSearch } from '@/utils/columnSearch';
--import { showDataFetchError, byteToSize } from '@/utils/common';
--import { AxiosGetHelper, cancelRequests } from '@/utils/axiosRequestHelper';
--import { IOption } from "@/components/multiSelect/multiSelect";
++import {ColumnSearch} from '@/utils/columnSearch';
++import {byteToSize, showDataFetchError} from '@/utils/common';
++import {AxiosGetHelper, cancelRequests} from '@/utils/axiosRequestHelper';
++import {IOption} from "@/components/multiSelect/multiSelect";
import './om.less';
@@@ -1104,4 -1111,4 +1111,4 @@@ export class Om extends React.Component
</div>
);
}
--}
++}
diff --cc hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx
index fff4a05a50,2b3aa2fcac..db30871a5a
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx
@@@ -19,13 -19,13 +19,13 @@@
import React from 'react';
import moment from 'moment';
import filesize from 'filesize';
--import { Table, Tooltip, Tabs } from 'antd';
--import { TablePaginationConfig } from 'antd/es/table';
--import { InfoCircleOutlined } from '@ant-design/icons';
++import {Table, Tabs, Tooltip} from 'antd';
++import {TablePaginationConfig} from 'antd/es/table';
++import {InfoCircleOutlined} from '@ant-design/icons';
--import { ColumnSearch } from '@/utils/columnSearch';
--import { showDataFetchError, timeFormat } from '@/utils/common';
--import { AxiosGetHelper, cancelRequests } from '@/utils/axiosRequestHelper';
++import {ColumnSearch} from '@/utils/columnSearch';
++import {showDataFetchError, timeFormat} from '@/utils/common';
++import {AxiosGetHelper, cancelRequests} from '@/utils/axiosRequestHelper';
import './missingContainers.less';
diff --cc hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
index 2085503079,c990815936..213f7fb4c2
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
@@@ -1098,9 -1078,8 +1101,8 @@@ public class TestContainerEndpoint
}
}
-
private void createUnhealthyRecords(int missing, int overRep, int underRep,
- int misRep) {
+ int misRep, int dataChecksum) {
int cid = 0;
for (int i = 0; i < missing; i++) {
createUnhealthyRecord(++cid, UnHealthyContainerStates.MISSING.toString(),
diff --cc hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java
index 2844ba9855,558b3cbebb..fc659b49fd
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java
@@@ -307,7 -315,15 +330,15 @@@ public class TestContainerHealthTask ex
// Just check once again that count remains consistent
LambdaTestUtils.await(60000, 1000, () ->
- (unHealthyContainersTableHandle.count() == 2));
+ (unHealthyContainersTableHandle.count() == 3));
+
+ // Since other container states have been changing, but no change in UNDER_REPLICATED
+ // container count, UNDER_REPLICATED count metric should not be affected from previous
+ // assertion count.
+ assertEquals(1, containerHealthMetrics.getUnderReplicatedContainerCount());
+ assertEquals(0, containerHealthMetrics.getMissingContainerCount());
+
+ containerHealthTask.stop();
}
@Test
diff --cc hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/datanode/container/ContainerCommands.java
index 7d36c56978,83f4ae3596..7912d88524
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/datanode/container/ContainerCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/datanode/container/ContainerCommands.java
@@@ -119,11 -121,10 +122,12 @@@ public class ContainerCommands extends
datanodeUuid,
containerSet,
volumeSet,
+ volumeChoosingPolicy,
metrics,
containerReplicaProto -> {
- });
+ },
+ // Since this is an Ozone debug CLI, this instance is not part of a running datanode.
+ new ContainerChecksumTreeManager(conf));
handler.setClusterID(clusterId);
handlers.put(containerType, handler);
}
diff --cc hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java
index 48ab8b6322,452846354a..ade3eac1f6
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java
@@@ -226,12 -228,11 +229,13 @@@ public class ClosedContainerReplicator
fakeDatanodeUuid,
containerSet,
volumeSet,
+ volumeChoosingPolicy,
metrics,
containerReplicaProto -> {
- });
- handler.setClusterID(clusterID);
+ },
+ // Since this a Freon tool, this instance is not part of a running datanode.
+ new ContainerChecksumTreeManager(conf));
+ handler.setClusterID(UUID.randomUUID().toString());
handlers.put(containerType, handler);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org