You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by er...@apache.org on 2026/08/26 18:25:29 UTC
(ozone) branch HDDS-14496-zdu updated (25495e1a9f5 -> 71a27e3fc1f)
This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a change to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 25495e1a9f5 HDDS-16190. Do not use unified version framework in OM until ZDU is finalized. (#11025)
add 1f515587d6f HDDS-9411. Restrict FileInputStream/FileOutputStream import (#10789)
add 458c0d14498 HDDS-15791. EC reconstructed RECOVERING container while still idle can be deleted by SCM and recreated as new partial OPEN container (#10702)
add 1bb35b2f1c1 HDDS-15892. Massive local build time degradation when using agent worktrees (#10791)
add 66e2bf5771d HDDS-15392. Add OmKeyInfo and OmDirectoryInfo to snapshot diff codec registry. (#10770)
add 769471cd9b0 HDDS-15093. Make RackScatter intra-rack placement capacity-aware (#10648)
add 7674aab192c HDDS-15899. Bump jackson to 2.21.5 (#10799)
add 7ea7e707ffe HDDS-15897. Bump awssdk to 2.47.5 (#10797)
add 96c6cc7e390 HDDS-15900. Bump netty to 4.1.136 (#10800)
add 2d2d63410f7 HDDS-15898. Bump orc-core to 1.9.9 (#10798)
add 9b7dccd80f2 HDDS-13255. Change `<? extends KeyValue>` to `<KeyValue>` in ozone-manger (#10793)
add 0694122a6f2 HDDS-15901. Bump actions/stale to 10.4.0 (#10802)
add ed92faaaa97 HDDS-15773. Make HddsUtils host:port helpers IPv6-safe (#10804)
add 472962558f2 HDDS-15902. Bump actions/setup-java to 5.5.0 (#10803)
add 81dcafaba03 HDDS-15772. Stop forcing java.net.preferIPv4Stack=true (#10801)
add 4766aa8609b HDDS-15440. Combine pipelineMap and pipeline2container to a single map in PipelineStateMap (#10454)
add 1115746a35a HDDS-15893. Change `<? extends KeyValue>` to `<KeyValue>` in remaining code (#10806)
add 7c54977e640 HDDS-15906. Add gateway LLM provider for Recon chatbot (OpenAI-compatible gateways). (#10805).
add 45b48e85823 HDDS-15740. Validate partNumber on GetObject and return InvalidPart for out-of-range parts (#10670).
add a3ca93ac435 HDDS-15342. Make RocksDB bottommost level compaction options configurable for background compaction (#10419)
add 66447d3663a HDDS-15690. Support Node-ID for DiskBalancer Commands (#10755).
add 0f6295ef3ab HDDS-15085. Add DN and Cluster Readiness (#10757)
add 8b167ca2732 HDDS-15877. Extend getFileStatus head-op optimization to o3fs and remaining type-only callers (#10779)
add c3de92cf9a0 HDDS-15324. Dashboard for monitoring Data Node performance
add c31553bcded HDDS-15766. Make Recon OM DB large tarball transfer reliable (#10696)
add 5bfa452f3e5 HDDS-15607. Return x-amz-meta-* user-defined object metadata on S3 GetObject (#10785)
add 917e4b77062 Merge branch 'HDDS-8342' into master-merge-object-ttl
add 1975e74d096 HDDS-15688. Retain in-memory PAUSED state when diskBalancer.info write fails on DN state change (#10818).
add 024509ede87 HDDS-15912. HttpFS Gateway launcher should use runtime-managed log/temp dirs instead of OZONE_HOME paths. (#10811)
add 8d61861f48f HDDS-15860. Always rewrite live SSTs before snapshot defrag ingestion (#10772)
add 2b830420d54 HDDS-15757. Streaming Write also commit PutBlock at the time of closing (#10694)
add 68c4231f512 Revert "Merge branch 'HDDS-8342' into master-merge-object-ttl"
add 738b9c60d33 HDDS-8342. S3 Object LifeCycle Management design doc. (#8240)
add 2b5f88aaf0b HDDS-12808. Implement OM entities of lifecycle configurations (#8275)
add f81b36593cc HDDS-12781. Implement LifecycleConfiguration proto (#8276)
add 9438be5596c HDDS-12974. Docker could not parse extra host IP (#8397) (#8545)
add 59a2b7e77d3 HDDS-12783. Implement S3Gateway api for LifeCycleConfiguration (#8557)
add c9ef5e66539 HDDS-12780. Implement the Lifecycle Service. (#8681)
add ce01a02542c HDDS-12782. Implement create, delete, and get for bucket lifecycle configurations in OM (#8875)
add fa33739da3e HDDS-12786. Refine keys lifecycle deletion batch size (#8909)
add 1fe66180880 HDDS-13548. Prevent OM startup failure from past lifecycle expiration dates (#8953)
add d7077ea0ce6 HDDS-12789. Support move expired key to trash if trash is enabled (#8974)
add 6542b31ff8a HDDS-12793. Recursive iterate FSO directory. (#9042)
add 3b5cb65c82a HDDS-12784. Add bucket object ID to lifecycle configuration proto definition (#9255)
add 99ebb223793 HDDS-12790. Mark a delete operation is performed by Lifecycle Service (#9403)
add 724803f21a0 HDDS-14184. Reject lifecycle configuration if 'Status' is missing (#9513)
add b16ea44f588 HDDS-12791. Make Retention Service suspendable (#9426)
add dd04f4b7d83 Merge remote-tracking branch 'origin/master' into HDDS-8342-ttl-branch
add cfa9cfee5a7 HDDS-12787. Make expired key move to trash configurable (#9594)
add 8ee502aa31b HDDS-12794. Improve Lifecycle rule validity checking (#9593)
add 949747727b3 HDDS-14377. Refactor OmLifecycleConfiguration#Builder (#9688)
add ff9446d6538 HDDS-14900. Document S3 Lifecycle Configurations - Object Expiration (#9979)
add c0706c6b99a HDDS-14347. Handle upgrades to support S3 lifecycle (#9614)
add 06ebc316b89 Merge remote-tracking branch 'origin/master' into HDDS-8342-ttl-branch
add f3c6ba4afae HDDS-13474. Support abort incomplete multipart upload action (#10052)
add f364114d112 Merge remote-tracking branch 'origin/master' into HDDS-8342
add fc4416274b3 HDDS-15227. Add robot test for Lifecycle (#10242)
add daef2f19c29 HDDS-15447. Persist bucket scanned key pointer periodically (#10412)
add 3d6aa1bd0fc Merge remote-tracking branch 'origin/master' into HDDS-8342-object-ttl
add 2ebf64de05c Merge remote-tracking branch 'origin/master' into HDDS-8342-object-ttl
add aa6377721c2 HDDS-15853. Fix Ranger ACL validation for lifecycle requests (#10753)
add cca31ff7206 Merge remote-tracking branch 'origin/HDDS-8342' into master
add c4b2ae1fff4 HDDS-15086. Wire ozone local command (#10758)
add 3b40495c46a HDDS-15782. [Recon] Poll DN pending-deletion scan to completion when Auto Refresh is off (#10714)
add fb3fdcc5f60 HDDS-13774. [Recon] Consolidate NSSummary endpoint tests into one suite (#10769)
add a57788ea4cd HDDS-12250. Speed up TestSecureOzoneCluster (#10834)
add 4417f6b7de2 HDDS-14665. Add upgrade handling to multipart requests (#10062)
add bb1da7fea63 HDDS-15764. [Recon] Show error card for unavailable datanode in Cluster Capacity (#10767)
add 77ceb2969b5 HDDS-15624. Resolve link bucket properties in listBucket. (#10745)
add 658f19df6c0 HDDS-14666. Add multipart upload split table read for Recon (#10847)
add 9d2d4adf55d HDDS-15959. Add title section to Resumable Lifecycle Scans design doc. (#10856)
add 72f1f9a612e HDDS-15608. Return BucketAlreadyOwnedByYou when the same S3 owner recreates an existing bucket (#10814).
add ed072a998f1 HDDS-15950. Fix NPE in S3 lifecycle GET for date-based expiration (#10850)
add a54c42d9413 HDDS-15888. Stamp lastTransactionInfo on snapshot deletion to avoid premature GC (#10788)
add 930f8fa4aa6 HDDS-15909. Refactor OzoneManagerLock. (#10852)
add 4f5ae5454fe HDDS-15388. K way merge iterator over native SST file reader to emit latest tombstone and latest value of a key. (#10484)
add 36ca073edfa HDDS-15970. Bump zizmorcore/zizmor-action to 0.6.0 (#10871)
add 6ec123e7089 HDDS-15692. Trigger GitHub Copilot code review only when PR is opened (#10751)
add 4726242563b HDDS-15974. Bump bouncycastle to 1.85 (#10867)
add f1c1cbfcb4b HDDS-10315. Speed up TestContainerReplication (#10858)
add 1b8f5f8de21 HDDS-15933. Remove usage of jersey internal ContainerRequest (#10839)
add f54483f7f47 HDDS-15971. Bump actions/setup-java to 5.6.0 (#10870)
add 7cb9ea1381d HDDS-15972. Bump actions/checkout to 7.0.1 (#10869)
add 92557fb3de5 HDDS-15813. Rename TestDataUtil to DataTestUtil (#10841)
add 65cb289fa14 HDDS-15896. Bump opentelemetry to 1.64.0 (#10796)
add 9c6d389c9a9 HDDS-15932. Remove usage of internal AWS S3 SDK API in S3KeyGenerator (#10835)
add 6cb24889013 HDDS-15973. Bump awssdk to 2.48.3 (#10868)
add 9fbf9ee0cb1 HDDS-15983. Bump actions/setup-python to 7.0.0 (#10874)
add c1875b844a3 HDDS-15934. Remove usage of internal jersey internal InjectionManager (#10853)
add 41b60a2a95d HDDS-15930. Avoid zero-length chunk in TestContainerOperations (#10833)
add 2473ec54d33 HDDS-15940. Handle unexpected OMExceptions in S3 putBucketLifecycleConfiguration (#10848)
add fcf0cc39a15 HDDS-15868. Enable dynamic agent loading for tests (#10816)
add dc6494ac7ae HDDS-15059. Shift streaming write sortDatanodes logic to OM (#10633)
add 1b00e7fb697 HDDS-15968. Exempt design pull requests from stale bot (#10864)
add ca0715a1a0a HDDS-15949. Handle split MPU part counting for abort batch sizing and container-key-mapping CLI (#10849)
add a481e65272a HDDS-15919. Fix broken Ozone Enhancement Proposals link in CONTRIBUTING.md (#10824).
add 273bf2f39b8 HDDS-15533. DNS refresh on heartbeat failure for DN to SCM (#10844)
add 0de9322992c HDDS-15922. Fix ozone-site config doc workflow reusing stale PR branch (#10826)
add d501066b60f HDDS-15922. (addendum) Fix ozone-site config doc workflow reusing stale PR branch (#10886)
add 8fd042079f4 HDDS-10314. Speed up TestDatanodeHddsVolumeFailureDetection (#10819)
add 7d9223cd5a9 HDDS-15920. Fix ByteBuffer positioned read behavior at EOF in OzoneFSInputStream (#10827)
add f19f2c14afb HDDS-15958. Return InvalidURI for unreadable S3 object keys on GetObject (#10860).
add eccb174c7b3 HDDS-15535. Container Balancer should validate configuration and report startup failures to user (#10812)
add eb183c6b5ac HDDS-16009. Switch zizmor check to ubuntu-24.04 (#10887)
add d9b6cd4fb31 HDDS-15863. Add Manual OM DB Rebuild Support for Recon Bootstrapping. (#10825).
add 247f5cab1f6 HDDS-15758. Commit PutBlock without Raft in Client (#10823)
add b22db56d10d HDDS-15969. Prioritize Ratis Streaming capable DataNodes during Pipeline creation with graceful fallback (#10865)
add 5ff8b53aa6b HDDS-15894. Add Table.clear for reusable table clearing (#10879)
add 8acc0413a95 HDDS-14812. Support datanode topology in MiniOzoneCluster (#10598)
add 147b009a6db HDDS-11622. Support domain socket creation (#7397)
add 46b858007e6 HDDS-11627. Support getBlock operation on short-circuit channel. (#7456)
add fb9398a3131 HDDS-11626. Document short-circuit read feature (#7529)
add 13a905468a2 Merge remote-tracking branch 'origin/master' into HDDS-10685
add cd3e1c7ca04 HDDS-12883. Close file descriptor of block file in datanode (#8319)
add 1395fae0640 Merge remote-tracking branch 'origin/master' into HDDS-10685
add 625a6ad85cd HDDS-11651. Support short-circuit read channel for freon rk command (#8316)
add 5676ac4b338 HDDS-15318. Download libhadoop library during maven build (#10305)
add 879707874bc Merge remote-tracking branch 'origin/master' into HDDS-10685
add 2f90c1731a3 HDDS-15404. Enable short-circuit unit test in CI (#10377)
add 6c9676481b9 HDDS-15442. Enhance short-circuit robot test (#10400)
add c2118e70585 Merge branch 'master' of github.com:apache/ozone into HDDS-10685
add 277737474cd HDDS-15559. Add libhadoop to DYLD_LIBRARY_PATH or LD_LIBRARY_PATH (#10513)
add 1ea7e9e7985 Merge remote-tracking branch 'origin/master' into HDDS-10685
add b01e0c715fa HDDS-15992. Convert short-circuit-read design PDF to markdown (#10877)
add 6db4545f1da HDDS-16013. Remove HDDSLayoutFeature.SHORT_CIRCUIT_READS layout version (#10893)
add ec381e5d3b0 Merge remote-tracking branch 'origin/HDDS-10685' into master
add 197db2d7edc HDDS-15558. Prevent addition of deprecated style CLI options (#10891)
add 3dbca34cdb4 HDDS-15606. Support S3 object metadata headers (Cache-Control, Expires, Content-Encoding) (#10786).
add d97e8d03486 HDDS-15990. Remove setExclusiveManualCompaction from CompactionService (#10876)
add bae32aac3c0 HDDS-16032. cd: lib/native: No such file or directory (#10909)
add 9da5de60b97 HDDS-15640. Bump msw to 2.15.0 and vitest to 3.2.7 (#10577)
add 14702aba6a0 HDDS-15963. NPE on EC degraded read in isolated ClassLoader env (#10861)
add 3a450fa0da3 HDDS-13803. Client aware tracing (#10477)
add e9b1308455f HDDS-16050. Fix aspectj profile activation upper bounds (#10912)
add 52980fec53b HDDS-15996. Flaky TestClientRetryTimeout throws NPE (#10898)
add d6012190454 HDDS-15907. Do not use Collection in IOzoneManagerLock. (#10907)
add 5197855a329 HDDS-12991. Recreate pipelines after enabling ratis write streaming (#10842)
add e39c536b753 HDDS-16031. Suppress java.util.logging in shell commands (#10910)
add 0c4c66e9f4b HDDS-15412. Add per-volume push replication thread pools on DataNode.
add 29ae1651322 HDDS-16063. Bump sqlite-jdbc to 3.53.2.1 (#10923)
add 3117c6805ac HDDS-16017. Bump axios version to 1.19.0 (#10905)
add 3055fcb9305 HDDS-16062. Bump jline to 3.30.16 (#10922)
add de1392809a6 HDDS-15814. Rename TestHelper to OzoneTestHelper (#10920)
add c628bfa4464 HDDS-16064. Bump maven-jar-plugin to 3.5.1 (#10924)
add bf103a657d3 HDDS-16061. Bump awssdk to 2.49.3 (#10921)
add 46456c3693a HDDS-16054. Bump zizmorcore/zizmor-action to 0.6.1 (#10925)
add ea74fada9cd HDDS-15777. Make admin CLI subcommands parse IPv6 RPC addresses correctly (#10885)
add 904acc05820 HDDS-15957. Clean up ozone-csi references after module removal (#10855)
add 3a18da72b56 HDDS-15935. Extract ExportFileManager and document container export directory layout (#10866)
add e697ef42161 HDDS-16059. Optimize varargs in IOzoneManagerLock. (#10919)
add cf8ae810cc1 HDDS-15645. ListObjectsV2 returns Owner when FetchOwner is not requested (#10897).
add 7bb7bcd956c HDDS-16005. Make DeleteBucketLifecycle idempotent on missing config (#10888)
add fb9f30b8869 HDDS-16057. OM bootstrap fails when checkpoint install is rejected during BOOTSTRAPPING. (#10917)
add 3492727f0f1 HDDS-16026. Restore the OM DB directory when the pre-install backup fails part way (#10904)
add 40cd903440e HDDS-16065. Fix flaky testCloseNonStreamablePipelineThenStream (#10927)
add 8415f3b8765 HDDS-13671. Reduce CLI calls in ozone admin container Robot test (#10947)
add 67a5e86db0a HDDS-16016. Adding unit test for datanodeUseHostname functionality in XceiverClientGrpc (#10896)
add 377ca2fe7a7 HDDS-15787. Log OM block deletion ack at INFO in SCMBlockProtocolServer (#10952)
add 4b8156b57d2 HDDS-9894. Handle IPv4 and IPv6 addresses invalid for CSR consistently (#10687)
add a2065ce9994 HDDS-16034. Certificates omit DNS SANs on clusters with non-public hostname suffixes (#10908)
add a152e6e34ba HDDS-16091. Recon fills disk with leaked checkpoints and crashes on startup when local OM DB is missing. (#10950)
add 7cca85d6ad1 HDDS-16043. Use EnumMap for ports in DatanodeDetails (#10948)
add 9ff43fef43f HDDS-16004. Return NotImplemented for GetObjectTorrent instead of GetObject body (#10899)
add b68bcf8b94d HDDS-16080. Reduce heartbeat interval in TestClientRetryContainerStateMachineFailures (#10958)
add ac4278ac630 HDDS-16090. Parameterize TestOzoneShellHA with follower read (#10954)
add 5fe740c2c12 HDDS-16047. Certificate renewal stops permanently after an unexpected failure (#10915)
add a01a27a38bd HDDS-13128. Expose per DN space utilisation as JMX metrics (#10913)
add 5ba550d965f HDDS-13173. Clean up temporary checkpoint directories on snapshot defrag failure (#10774)
add 036745a326d HDDS-15962. Remove leader readiness check on the bootstrap flow. (#10862)
add 54facbe255e HDDS-16056. non-varargs call of varargs method with inexact argument type (#10914)
add 951ed5cb51b HDDS-16086. installCheckpoint sets OM transaction info inconsistent with the checkpoint index (#10943)
add 3b8c05e635e HDDS-13217. Test that snapshot checkpoint content is preserved after defrag (#10951)
add 80ffb62a841 HDDS-16066. replicas after re-replication should span at least 2 racks (#10944)
add cb6f12408ec HDDS-15852. Allow CopyObject to the same key when the metadata directive is REPLACE (#10752).
add fce67f62f35 HDDS-16096. Bump Ranger to 2.9.0 (#10957)
add c8fa8dafd5c HDDS-16101. Extract bucket lifecycle operations into dedicated handler (#10969)
add af3e34377b0 HDDS-16076. Separate test class for testInstallSnapshotDuringBootstrapping. (#10931)
add 0a9bd0d999a HDDS-16067. Intermittent timeout in testNodeWithOpenPipelineCanBeDecommissionedAndRecommissioned (#10939)
add 5425ee3f86c HDDS-15781. Adding Spankind information to ozone sh commands (#10695)
add 0110ff57df4 HDDS-15396. Grafana dashboard for disk balancer (#10609).
add bd0f23b49ba HDDS-16119. Fix DatanodeStorageMetrics metrics-system deadlock and mini-cluster source leak (#10988)
add 5be6afae7af HDDS-15389. Top-level delete entry retention utilities for snapshot diff. (#10750)
add 5e99e226bb0 HDDS-16081. TestOMRatisSnapshots fails due to leader applied index taken before flush (#10959)
add 09a7d615dec HDDS-15982. Fix UnsupportedOperationException and add idempotent deletion in RangerClientMultiTenantAccessController (#10872)
add 07f069b72fc HDDS-16015. Make listStatusLight skip SCM pipeline refresh. (#10894)
add 6384c643fb3 HDDS-14742. Clean up orphan files from a terminated compaction. (#10821)
add b4a4227cf4f HDDS-14876. Make snapshot diff progress more granular (#10901)
add 776ce13a798 HDDS-16113. Add regression tests for snapshot defragmentation cleanup (#10978)
add a1ce3b9d900 HDDS-15774. Make SCM Ratis roles string parsing IPv6-safe (#10831)
add 5dfed2ce426 HDDS-15775. Support IPv6 literal authorities in OzoneFS URI parsing (#10933)
add 752e96c2fb3 HDDS-15776. Make S3 Gateway VirtualHostStyleFilter IPv6-safe (#10809)
add 0f812bcbb30 HDDS-15542. Closed-container reconciliation advances BCSID past a hole, masking missing chunks (#10496)
add 9065c84364d HDDS-15761. Create test for OMMXBean (#10960)
add 59351d77aba HDDS-16114. Avoid misleading Ozone mount warning in acceptance test results (#10979)
add 1db063b63ce HDDS-16072. Implement GetObjectAttributes: ETag, ObjectSize, StorageClass, and ObjectParts count (#10930).
add 291633510ce HDDS-15961. Resolve linked bucket source properties consistently (#10859)
add a7f5cbc832c HDDS-16117. Failed FSO multipart complete leaks bucket namespace quota (#10984)
add 2fbc910aa29 HDDS-11620. Log SCM finalization completion (#10963)
add 6a4513ddaec HDDS-15826. Recon: add cycle guard to NSSummary /du tree walks (#10723)
add 5672fc71c5d HDDS-8082. Check for unnecessary newlines in config defaults (#10840)
add 7f68eb6f36d HDDS-16125. Copy bucket snapshot in quota repair response to fix cache aliasing (#11004)
add bc21296f297 HDDS-16118. Release snapshot DB handles before KeyDeletingService submits OM requests (#10987)
add 7693b1979f8 HDDS-15895. Make ServerNotLeaderException suggested leader flow IPv6-safe (#10807)
add 0c349a3f541 HDDS-15356. Make multi-buffer chunk checksum allocation-free (#10350)
add ce13cdece18 HDDS-15911. Recon AI Assistant: add conversation memory for follow-up questions. (#10808)
add 749da668af1 HDDS-16158. Fix sendDeleteKeysRequestAndClearList Authorization and FSO Trash Handling (#10996)
add fb2cc733688 HDDS-16169. Add old updateContainerState to handle upgrade path. (#11012)
add 711476cb0c4 HDDS-16089. Improve MPU table evaluation efficiency in KeyLifeCycleService (#10949)
add 50978e3ad7c HDDS-15422. Stream read seek should not close stream (#10415)
new 71a27e3fc1f Merge branch 'master' into HDDS-14496-zdu
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 1 -
.github/workflows/asf-allowlist-check.yaml | 2 +-
.github/workflows/build-ratis.yml | 6 +-
.github/workflows/check.yml | 4 +-
.github/workflows/ci.yml | 12 +-
.github/workflows/close-stale-prs.yaml | 3 +-
.github/workflows/generate-config-doc.yml | 4 +-
.github/workflows/intermittent-test-check.yml | 10 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/populate-cache.yml | 6 +-
.github/workflows/pull-request.yml | 2 +-
.github/workflows/repeat-acceptance.yml | 8 +-
.github/workflows/update-ozone-site-config-doc.yml | 57 +-
.github/workflows/zizmor.yml | 6 +-
AGENTS.md | 2 +-
CONTRIBUTING.md | 2 +-
THREAT_MODEL.md | 16 +-
.../ozone/annotations/CliOptionStyleProcessor.java | 123 +
hadoop-hdds/cli-common/pom.xml | 11 +
hadoop-hdds/client/pom.xml | 8 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 68 +
.../hadoop/hdds/scm/XceiverClientCreator.java | 37 +-
.../hadoop/hdds/scm/XceiverClientFactory.java | 19 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 13 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 103 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 3 +-
.../hadoop/hdds/scm/XceiverClientShortCircuit.java | 623 ++
.../hdds/scm/storage/BlockDataStreamOutput.java | 28 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 149 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 16 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +-
.../apache/hadoop/hdds/scm/storage/DomainPeer.java | 111 +
.../hdds/scm/storage/DomainSocketFactory.java | 276 +
.../hdds/scm/storage/ECBlockOutputStream.java | 25 +-
.../hdds/scm/storage/LocalChunkInputStream.java | 116 +
.../hdds/scm/storage/MultipartInputStream.java | 12 +-
.../hdds/scm/storage/StreamBlockInputStream.java | 186 +-
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 17 +
.../hdds/scm/TestXceiverClientGrpcChannel.java | 68 +
.../hdds/scm/client/TestHddsClientUtils.java | 37 +
.../hdds/scm/storage/MockDatanodePipeline.java | 29 +-
.../scm/storage/TestBlockDataStreamOutput.java | 22 +-
.../storage/TestBlockOutputStreamCorrectness.java | 52 +-
hadoop-hdds/common/src/main/conf/ozone-env.sh | 8 +-
.../java/org/apache/hadoop/hdds/HDDSVersion.java | 1 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 4 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 85 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 74 +-
.../hdds/ratis/ServerNotLeaderException.java | 5 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 3 +
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 14 +
.../balancer/ContainerBalancerConfiguration.java | 17 +-
.../apache/hadoop/hdds/scm/net/HostAndPort.java | 23 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 1 +
.../ContainerCommandResponseBuilders.java | 20 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 65 +-
.../hadoop/hdds/tracing/GrpcServerInterceptor.java | 4 +-
.../{SkipTracing.java => InternalSpanKind.java} | 6 +-
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 9 +-
.../apache/hadoop/hdds/tracing/TracingConfig.java | 16 +
.../apache/hadoop/hdds/tracing/TracingUtil.java | 245 +-
.../org/apache/hadoop/hdds/utils/CompositeKey.java | 109 +-
.../apache/hadoop/hdds/utils/SimpleStriped.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../org/apache/hadoop/ozone/common/Checksum.java | 200 +-
.../apache/hadoop/ozone/common/ChecksumCache.java | 117 +-
.../hadoop/ozone/common/utils/BufferUtils.java | 23 +
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 24 +-
.../org/apache/hadoop/ozone/util/UUIDUtil.java | 9 +
.../common/src/main/resources/ozone-default.xml | 114 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 90 +
.../hadoop/hdds/protocol/TestDatanodeDetails.java | 37 +
...TestServerNotLeaderExceptionMessageParsing.java | 22 +
.../hadoop/hdds/scm/net/TestHostAndPort.java | 62 +
.../hadoop/hdds/tracing/TestTraceAllMethod.java | 20 +
.../hdds/tracing/TestTracingAnnotations.java | 88 +
.../hadoop/hdds/tracing/TestTracingInitModes.java | 193 +
.../hadoop/hdds/tracing/TestTracingUtil.java | 77 +-
.../apache/hadoop/hdds/utils/TestCompositeKey.java | 108 +
.../hadoop/hdds/utils/TestProtobufUtils.java | 36 +
.../hadoop/hdds/utils/TestSimpleStriped.java | 3 +-
.../apache/hadoop/ozone/common/TestChecksum.java | 17 +
.../hadoop/ozone/common/TestChecksumCache.java | 73 +-
.../ozone/common/TestChecksumMultiBuffer.java | 97 +
.../hadoop/ozone/common/utils/TestBufferUtils.java | 81 +
.../ozone/container/ContainerTestHelper.java | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 28 +-
.../org/apache/hadoop/ozone/audit/DNAction.java | 1 +
.../checksum/ContainerChecksumTreeManager.java | 12 +-
.../container/checksum/ContainerDiffReport.java | 13 +
.../container/common/helpers/ContainerMetrics.java | 49 +-
.../container/common/helpers/ContainerUtils.java | 26 +
.../ozone/container/common/impl/ContainerSet.java | 95 +-
.../container/common/impl/HddsDispatcher.java | 20 +-
.../common/interfaces/ContainerDispatcher.java | 6 +-
.../ozone/container/common/interfaces/Handler.java | 27 +-
.../common/statemachine/DatanodeStateMachine.java | 7 +
.../common/statemachine/EndpointStateMachine.java | 10 +-
.../common/statemachine/SCMConnectionManager.java | 98 +-
.../states/endpoint/HeartbeatEndpointTask.java | 32 +
.../common/transport/server/Receiver.java | 353 +
.../server/XceiverServerDomainSocket.java | 324 +
.../common/transport/server/XceiverServerGrpc.java | 57 +-
.../common/transport/server/XceiverServerSpi.java | 2 +
.../server/ratis/ContainerStateMachine.java | 20 +-
.../transport/server/ratis/XceiverServerRatis.java | 9 +-
.../common/volume/DatanodeStorageMetrics.java | 123 +
.../container/common/volume/MutableVolumeSet.java | 37 +
.../diskbalancer/DiskBalancerService.java | 17 +-
.../ECReconstructionCoordinator.java | 3 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 246 +-
.../container/keyvalue/helpers/BlockUtils.java | 9 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 24 +
.../keyvalue/impl/ChunkManagerDispatcher.java | 15 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 30 +-
.../keyvalue/impl/KeyValueStreamDataChannel.java | 104 +-
.../keyvalue/impl/StreamDataChannelBase.java | 16 +
.../keyvalue/interfaces/BlockManager.java | 10 +
.../keyvalue/interfaces/ChunkManager.java | 22 +-
.../StaleRecoveringContainerScrubbingService.java | 21 +-
.../ozone/container/metadata/DatanodeTable.java | 12 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 82 +-
.../container/replication/ReplicationServer.java | 73 +-
.../replication/ReplicationSupervisor.java | 183 +-
.../replication/ReplicationSupervisorMetrics.java | 5 +-
.../replication/VolumeReplicationThreadPools.java | 166 +
.../ContainerTableSchemaFinalizeAction.java | 2 +-
.../TestSchemaOneBackwardsCompatibility.java | 19 +-
...stStaleRecoveringContainerScrubbingService.java | 50 +
.../container/common/impl/TestHddsDispatcher.java | 75 +
.../statemachine/TestSCMConnectionManager.java | 91 +
.../TestHeartbeatEndpointTaskDnsRefresh.java | 136 +
.../common/volume/TestDatanodeStorageMetrics.java | 162 +
.../container/common/volume/TestVolumeSet.java | 27 +
.../diskbalancer/TestDiskBalancerService.java | 43 +
.../container/keyvalue/TestKeyValueHandler.java | 2 +-
.../TestReconcileChunksPerBlockHoleBcsId.java | 522 ++
.../keyvalue/impl/TestFilePerBlockStrategy.java | 10 +-
.../impl/TestKeyValueStreamDataChannel.java | 139 +-
.../replication/TestReplicationConfig.java | 30 +
.../replication/TestReplicationSupervisor.java | 473 +-
.../upgrade/TestDatanodeVersionManager.java | 2 +-
hadoop-hdds/docs/content/design/diskbalancer.md | 12 +-
.../docs/content/design/lifecycle-task-resume.md | 85 +
...ocksdb.md => s3-object-lifecycle-management.md} | 18 +-
.../design/short-circuit-read-current-flow.png | Bin 0 -> 33740 bytes
.../design/short-circuit-read-getblock-flow.png | Bin 0 -> 66181 bytes
.../design/short-circuit-read-hbase-benchmark.png | Bin 0 -> 34029 bytes
.../short-circuit-read-mmap-cache-benchmark.png | Bin 0 -> 28688 bytes
.../design/short-circuit-read-target-flow.png | Bin 0 -> 39362 bytes
.../short-circuit-read-unix-domain-socket.png | Bin 0 -> 82958 bytes
.../docs/content/design/short-circuit-read.md | 175 +
hadoop-hdds/docs/content/feature/Decommission.md | 9 +-
hadoop-hdds/docs/content/feature/DiskBalancer.md | 37 +-
.../docs/content/feature/DiskBalancer.zh.md | 34 +-
hadoop-hdds/docs/content/feature/Lifecycle.md | 397 +
hadoop-hdds/docs/content/feature/Lifecycle.zh.md | 396 +
.../docs/content/feature/Reconfigurability.md | 3 +-
.../docs/content/feature/Short-Circuit-Read.md | 90 +
.../docs/content/feature/Short-Circuit-Read.zh.md | 88 +
.../docs/content/interface/pvtest-webui.png | Bin 13830 -> 0 bytes
hadoop-hdds/docs/content/tools/_index.md | 2 +
.../apache/ozone/erasurecode/CodecRegistry.java | 3 +-
.../TestCodecRegistryTcclIsolation.java | 52 +
.../hadoop/hdds/scm/client/ScmTopologyClient.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 4 +-
.../scm/proxy/SCMFailoverProxyProviderBase.java | 95 +-
.../authority/profile/DefaultProfile.java | 4 +-
.../client/DefaultCertificateClient.java | 11 +
.../certificate/utils/CertificateSignRequest.java | 46 +-
.../security/x509/certificate/utils/DnsNames.java | 107 +
.../certificate/utils/SelfSignedCertificate.java | 59 +-
.../server/OzoneProtocolMessageDispatcher.java | 3 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 37 +-
.../hadoop/hdds/utils/RDBSnapshotProvider.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 28 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 9 +
.../TestSCMFailoverProxyProviderRefreshWired.java | 167 +-
.../certificate/authority/TestDefaultCAServer.java | 45 +
.../certificate/authority/TestDefaultProfile.java | 42 +
.../client/TestDefaultCertificateClient.java | 50 +
.../utils/TestCertificateSignRequest.java | 125 +
.../x509/certificate/utils/TestDnsNames.java | 93 +
.../certificate/utils/TestRootCertificate.java | 47 +
.../hadoop/hdds/utils/db/InMemoryTestTable.java | 5 +
.../hadoop/hdds/utils/db/TestTypedTable.java | 48 +
.../src/main/proto/DatanodeClientProtocol.proto | 16 +
.../dev-support/findbugsExcludeFile.xml | 9 +
hadoop-hdds/rocks-native/pom.xml | 6 +-
.../utils/db/LatestVersionedKWayMergeIterator.java | 504 ++
.../db/TestLatestVersionedKWayMergeIterator.java | 253 +
...estLatestVersionedKWayMergeIteratorOverSst.java | 204 +
.../hdds/utils/db/TestRawSstFileRecords.java | 121 +
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 48 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 55 +-
hadoop-hdds/server-scm/pom.xml | 6 +
.../hdds/scm/container/ContainerStateManager.java | 6 +
.../scm/container/ContainerStateManagerImpl.java | 30 +
.../scm/container/balancer/ContainerBalancer.java | 158 +
.../container/balancer/ContainerBalancerTask.java | 16 +-
.../scm/container/export/ExportFileManager.java | 234 +
.../hdds/scm/container/export/ExportJob.java | 74 +
.../hdds/scm/container/export/ExportScope.java | 68 +
.../hdds/scm/container/export}/package-info.java | 4 +-
.../SCMContainerPlacementRackScatter.java | 49 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 2 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 31 +-
.../ha/invoker/ContainerStateManagerInvoker.java | 25 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 5 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 68 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 2 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 263 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 55 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 9 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 26 +
.../container/balancer/TestContainerBalancer.java | 107 +-
.../container/export/TestExportFileManager.java | 156 +
.../TestSCMContainerPlacementRackScatter.java | 34 +
.../hadoop/hdds/scm/ha/TestSCMRatisServerImpl.java | 62 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 76 +
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 187 +
.../scm/pipeline/TestRatisPipelineProvider.java | 90 +-
.../scm/server/upgrade/TestScmVersionManager.java | 2 +-
hadoop-ozone/cli-admin/pom.xml | 15 +
.../scm/cli/ContainerBalancerStartSubcommand.java | 2 -
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 49 +-
.../datanode/AbstractDiskBalancerSubCommand.java | 162 +-
.../hdds/scm/cli/datanode/DatanodeParameters.java | 3 +-
.../scm/cli/datanode/DiskBalancerCommands.java | 31 +-
.../cli/datanode/DiskBalancerCommonOptions.java | 12 +
.../cli/datanode/DiskBalancerReportSubcommand.java | 37 +-
.../cli/datanode/DiskBalancerStatusSubcommand.java | 48 +-
.../cli/datanode/DiskBalancerSubCommandUtil.java | 139 +-
.../ozone/admin/nssummary/NSSummaryAdmin.java | 31 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 3 +-
.../ozone/admin/om/LifecycleResumeSubCommand.java} | 50 +-
.../ozone/admin/om/LifecycleStatusSubCommand.java | 88 +
.../LifecycleSubCommand.java} | 26 +-
.../ozone/admin/om/LifecycleSuspendSubCommand.java | 75 +
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 3 +-
.../admin/scm/GetScmRatisRolesSubcommand.java | 51 +-
.../datanode/TestContainerBalancerSubCommand.java | 4 +-
.../datanode/TestDiskBalancerSubCommandUtil.java | 135 +
.../cli/datanode/TestDiskBalancerSubCommands.java | 345 +-
.../ozone/scm/TestGetScmRatisRolesSubcommand.java | 65 +
hadoop-ozone/cli-debug/pom.xml | 11 +
.../ozone/debug/om/ContainerToKeyMapping.java | 82 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 3 +-
.../ozone/debug/om/TestContainerToKeyMapping.java | 59 +
hadoop-ozone/cli-repair/pom.xml | 11 +
.../schemaupgrade/UpgradeContainerSchema.java | 3 +-
.../ozone/repair/ldb/RocksDBManualCompaction.java | 15 +-
.../apache/hadoop/ozone/repair/om/CompactOMDB.java | 14 +-
.../hadoop/ozone/repair/om/FSORepairTool.java | 25 +-
.../hadoop/ozone/repair/ldb/TestLdbRepair.java | 2 +-
hadoop-ozone/cli-shell/pom.xml | 15 +
.../java/org/apache/hadoop/ozone/shell/Shell.java | 3 +-
.../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 4 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 7 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 34 +
.../hadoop/ozone/client/OzoneClientFactory.java | 5 +-
.../ozone/client/OzoneLifecycleConfiguration.java | 230 +
.../ozone/client/protocol/ClientProtocol.java | 45 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 60 +-
.../ozone/client/MockXceiverClientFactory.java | 9 +-
.../client/checksum/FileChecksumBenchmark.java | 12 +
hadoop-ozone/common/pom.xml | 4 +
.../apache/hadoop/fs/ozone/OzoneTrashPolicy.java | 2 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 18 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 53 +
.../java/org/apache/hadoop/ozone/om/OmConfig.java | 16 +
.../hadoop/ozone/om/exceptions/OMException.java | 3 +
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 26 +
.../OmLCAbortIncompleteMultipartUpload.java | 139 +
.../apache/hadoop/ozone/om/helpers/OmLCAction.java | 59 +
.../hadoop/ozone/om/helpers/OmLCExpiration.java | 215 +
.../apache/hadoop/ozone/om/helpers/OmLCFilter.java | 235 +
.../apache/hadoop/ozone/om/helpers/OmLCRule.java | 411 +
.../ozone/om/helpers/OmLifecycleConfiguration.java | 334 +
.../om/helpers/OmLifecycleRuleAndOperator.java | 214 +
.../ozone/om/helpers/OmLifecycleScanState.java | 255 +
.../hadoop/ozone/om/helpers/OmLifecycleUtils.java | 131 +
.../ozone/om/helpers/OmMultipartKeyInfo.java | 30 +-
.../ozone/om/helpers/OmMultipartPartInfo.java | 32 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 72 +
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 3 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 65 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 116 +-
.../ozone/snapshot/SnapshotDiffResponse.java | 28 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 19 +
.../apache/hadoop/ozone/om/helpers/OMLCUtils.java | 131 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 71 +
.../TestOmLCAbortIncompleteMultipartUpload.java | 171 +
.../ozone/om/helpers/TestOmLCExpiration.java | 198 +
.../hadoop/ozone/om/helpers/TestOmLCFilter.java | 159 +
.../hadoop/ozone/om/helpers/TestOmLCRule.java | 459 +
.../om/helpers/TestOmLifeCycleConfiguration.java | 266 +
.../om/helpers/TestOmLifecycleRuleAndOperator.java | 139 +
.../ozone/om/helpers/TestOmLifecycleScanState.java | 79 +
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 22 +-
.../ozone/snapshot/TestSnapshotDiffResponse.java | 40 +-
.../dist/dev-support/bin/dist-layout-stitching | 23 +-
.../Datanode Chunk Read_Write Dashboard.json | 1601 ----
.../dashboards/Ozone - DataNode Overview.json | 9417 ++++++++++++++++++++
.../grafana/dashboards/Ozone - Disk Balancer.json | 1096 +++
.../dist/src/main/compose/ozone-ha/docker-config | 26 +
.../dist/src/main/compose/ozone/docker-config | 14 +-
.../diskbalancer.yaml => ozone/short-circuit.yaml} | 11 +-
.../{test-failures1.sh => test-short-circuit.sh} | 17 +-
.../src/main/compose/ozonesecure-ha/docker-config | 6 +
...st-s3g-virtual-host.sh => test-om-lifecycle.sh} | 10 +-
.../compose/ozonesecure-ha/test-repair-tools.sh | 2 +-
.../src/main/compose/ozonesecure/docker-config | 26 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 4 +-
.../src/main/smoketest/admincli/container.robot | 14 +-
.../src/main/smoketest/freon/read-write-key.robot | 7 +
.../main/smoketest/lifecycle/om-lifecycle.robot | 119 +
.../dist/src/main/smoketest/s3/bucketcreate.robot | 2 +-
.../src/main/smoketest/s3/bucketlifecycle.robot | 63 +
.../dist/src/main/smoketest/s3/objectputget.robot | 10 +-
.../smoketest/short-circuit/short-circuit.robot | 55 +
hadoop-ozone/dist/src/main/smoketest/testlib.sh | 3 +-
.../conf/shell-logging.properties} | 2 +
hadoop-ozone/dist/src/shell/ozone/ozone | 8 +-
.../dist/src/shell/ozone/ozone-functions.sh | 55 +-
...tions_test_helper.bash => ozone_bootstrap.bats} | 30 +-
hadoop-ozone/freon/pom.xml | 11 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 34 +
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 3 +-
.../apache/ozone/fs/http/server/HttpFSServer.java | 2 +-
hadoop-ozone/iceberg/pom.xml | 20 +-
hadoop-ozone/insight/pom.xml | 17 +-
.../ozone/insight/BaseInsightSubCommand.java | 35 +-
.../ozone/insight/TestBaseInsightSubCommand.java | 15 +
.../AbstractTestStorageDistributionEndpoint.java | 2 +
.../hadoop/ozone/recon/ReconEndpointTestUtil.java | 25 +-
.../ozone/recon/TestNSSummaryMemoryLeak.java | 4 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 14 +-
.../TestReconInsightsForDeletedDirectories.java | 8 +-
.../ozone/recon/TestReconWithOzoneManager.java | 116 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 4 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 9 -
hadoop-ozone/integration-test-s3/pom.xml | 12 +-
.../hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java | 11 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 443 +-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 584 ++
hadoop-ozone/integration-test/pom.xml | 16 +-
.../hadoop/conf/ConfigurationFieldsTests.java | 1 -
.../fs/ozone/AbstractOzoneFileSystemTest.java | 18 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 38 +-
.../AbstractRootedOzoneFileSystemTestWithFSO.java | 3 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 14 +-
.../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 4 +-
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 6 +-
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 4 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 149 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 8 +-
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 2 +-
.../TestOzoneFileSystemDataStreamEnablement.java | 432 +
.../fs/ozone/TestOzoneFileSystemMetrics.java | 4 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 4 +-
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 4 +-
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 4 +-
...oneFileSystemWithStreamingDisabledDatanode.java | 4 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 4 +-
.../hadoop/hdds/TestXceiverServerDomainSocket.java | 793 ++
.../apache/hadoop/hdds/scm/TestCloseContainer.java | 12 +-
.../hadoop/hdds/scm/TestContainerOperations.java | 2 +-
.../hadoop/hdds/scm/TestFailoverWithSCMHA.java | 1 +
.../hadoop/hdds/scm/TestRackAwarePlacement.java | 438 +
.../TestSCMFollowerCatchupWithContainerReport.java | 4 +-
.../hdds/scm/TestStorageContainerManager.java | 12 +-
...estStorageContainerManagerHAWithAllRunning.java | 4 +-
.../apache/hadoop/hdds/scm/TestWatchForCommit.java | 12 +-
.../hadoop/hdds/scm/TestXceiverClientManager.java | 6 +-
.../hdds/scm/TestXceiverClientManagerSC.java | 109 +
.../TestPendingContainerTrackerIntegration.java | 4 +-
.../metrics/TestSCMContainerManagerMetrics.java | 4 +-
.../TestReplicationManagerIntegration.java | 12 +-
.../scm/node/TestDecommissionAndMaintenance.java | 11 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 8 +-
.../hdds/scm/safemode/TestSafeModeSCMHA.java | 4 +-
.../TestDNDataDistributionFinalization.java | 3 +-
.../TestScmDataDistributionFinalization.java | 8 +-
.../ozone/{TestDataUtil.java => DataTestUtil.java} | 6 +-
.../ozone/TestContainerBalancerOperations.java | 24 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 66 +-
.../apache/hadoop/ozone/TestOMSortDatanodes.java | 52 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 58 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 1013 +--
.../ozone/admin/om/lease/TestLeaseRecoverer.java | 4 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 267 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../client/rpc/TestBlockDataStreamOutput.java | 63 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 6 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 12 +-
...stClientRetryContainerStateMachineFailures.java | 3 -
.../ozone/client/rpc/TestClientRetryTimeout.java | 50 +-
.../rpc/TestCloseContainerHandlingByClient.java | 8 +-
.../rpc/TestContainerReplicationEndToEnd.java | 4 +-
.../client/rpc/TestContainerStateMachine.java | 4 +-
.../rpc/TestContainerStateMachineFailures.java | 28 +-
.../rpc/TestContainerStateMachineStream.java | 34 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 4 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 8 +-
.../client/rpc/TestFailureHandlingByClient.java | 12 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 6 +-
.../client/rpc/TestHybridPipelineOnDatanode.java | 6 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 6 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 8 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 154 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 8 +-
.../rpc/TestOzoneClientRetriesOnExceptions.java | 10 +-
.../TestOzoneRpcClientWithKeyLatestVersion.java | 4 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 4 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 6 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 10 +-
.../ozone/client/rpc/read/InputStreamTests.java | 26 +-
.../client/rpc/read/TestChunkInputStream.java | 16 +-
.../client/rpc/read/TestDomainSocketFactory.java | 81 +
.../ozone/client/rpc/read/TestKeyInputStream.java | 6 +-
.../client/rpc/read/TestLocalChunkInputStream.java | 163 +
.../rpc/read/TestStreamBlockInputStream.java | 56 +-
.../ozone/client/rpc/read/TestStreamRead.java | 1 +
.../rpc/read/TestStreamReadDatanodeFailover.java | 6 +-
.../{TestHelper.java => OzoneTestHelper.java} | 6 +-
.../ozone/container/TestContainerReplication.java | 112 +-
.../container/TestContainerReportHandling.java | 12 +-
.../commandhandler/TestBlockDeletion.java | 6 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
.../commandhandler/TestFinalizeBlock.java | 3 +-
.../dn/TestDatanodeStorageMetricsIntegration.java | 134 +
.../TestContainerCommandReconciliation.java | 15 +-
...estDuplicateContainerDirScannerIntegration.java | 6 +-
.../scanner/ContainerScannerIntegrationTests.java | 6 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 441 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 4 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 4 +-
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 4 +-
.../ozone/local/TestLocalOzoneClusterRuntime.java | 34 +-
.../apache/hadoop/ozone/om/BucketForTesting.java | 4 +-
.../org/apache/hadoop/ozone/om/OMMXBeanTests.java | 88 +
.../ozone/om/TestBucketLayoutWithOlderClient.java | 8 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 2 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 12 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 4 +-
.../org/apache/hadoop/ozone/om/TestListKeys.java | 8 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 6 +-
.../org/apache/hadoop/ozone/om/TestListStatus.java | 4 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 8 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 18 +-
.../TestOMInstallSnapshotDuringBootstrapping.java | 207 +
.../ozone/om/TestOMRatisSnapshotTransfer.java | 2 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 193 +-
.../ozone/om/TestOMStartupWithBucketLayout.java | 14 +-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 4 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 8 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 4 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 30 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 18 +-
.../ozone/om/TestOmContainerLocationCache.java | 4 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 20 +-
.../om/TestOmStartupInvalidApparentVersion.java | 4 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 10 +-
.../ozone/om/service/TestBlockDeletionService.java | 6 +-
.../TestDirectoryDeletingServiceWithFSO.java | 4 +-
.../ozone/om/service/TestRootedDDSWithFSO.java | 13 +-
...TestSnapshotDeletingServiceIntegrationTest.java | 38 +-
.../hadoop/ozone/om/snapshot/OmSnapshotTests.java | 53 +-
.../TestOmSnapshotCheckpointDbContent.java | 511 ++
.../om/snapshot/TestOmSnapshotFileSystem.java | 6 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 6 +-
.../snapshot/TestSnapshotBackgroundServices.java | 2 +-
.../TestSnapshotDirectoryCleaningService.java | 6 +-
.../reconfig/TestDatanodeReconfiguration.java | 2 +
.../shell/TestOzoneContainerUpgradeShell.java | 6 +-
.../ozone/shell/TestOzoneDebugReplicasVerify.java | 2 +-
.../hadoop/ozone/shell/TestOzoneDebugShell.java | 6 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 185 +-
.../ozone/shell/TestOzoneShellHAWithFSO.java | 43 -
.../shell/TestOzoneShellHAWithFollowerRead.java | 42 +-
.../shell/TestReplicationConfigPreference.java | 12 +-
.../java/org/apache/ozone/test/NonHATests.java | 9 +
.../src/main/proto/OmClientProtocol.proto | 193 +-
.../interface-client/src/main/resources/proto.lock | 18 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 30 +-
.../hadoop/ozone/om/lock/IOzoneManagerLock.java | 46 +-
.../org/apache/hadoop/ozone/FixedHostMapping.java | 153 +
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 50 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 92 +
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 17 +-
.../hadoop/ozone/UniformDatanodesFactory.java | 6 +
.../RangerClientMultiTenantAccessController.java | 98 +-
...lientMultiTenantAccessControllerMockClient.java | 153 +
hadoop-ozone/ozone-manager/pom.xml | 11 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 8 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 64 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 201 +-
.../org/apache/hadoop/ozone/om/ListIterator.java | 3 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 8 +-
.../om/OMDBCheckpointServletInodeBasedXfer.java | 8 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 56 +
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 4 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 7 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 104 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 37 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 32 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 4 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 342 +-
.../hadoop/ozone/om/SnapshotChainManager.java | 2 +-
.../hadoop/ozone/om/SstFilteringService.java | 3 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 1 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 47 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 2 +-
.../hadoop/ozone/om/helpers/OMAuditLogger.java | 1 +
.../ozone/om/lock/DAGResourceLockTracker.java | 5 +
.../ozone/om/lock/LeveledResourceLockTracker.java | 6 +
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 1 +
.../hadoop/ozone/om/lock/OmReadOnlyLock.java | 58 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 512 +-
.../hadoop/ozone/om/lock/ResourceLockTracker.java | 2 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 12 +
.../ozone/om/request/OMClientRequestUtils.java | 2 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 18 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 4 +-
.../ozone/om/request/file/OMFileRequest.java | 2 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 2 +
.../om/request/key/OMKeyRenameRequestWithFSO.java | 8 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 62 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 109 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 6 +-
.../OMLifecycleConfigurationDeleteRequest.java | 209 +
.../OMLifecycleConfigurationSetRequest.java | 238 +
.../lifecycle/OMLifecycleSaveScanStateRequest.java | 74 +
.../OMLifecycleSetServiceStatusRequest.java | 125 +
.../ozone/om/request/lifecycle}/package-info.java | 4 +-
.../S3InitiateMultipartUploadRequest.java | 39 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 3 +
.../S3MultipartUploadCommitPartRequest.java | 18 +-
.../S3MultipartUploadCompleteRequest.java | 7 +
.../request/snapshot/OMSnapshotDeleteRequest.java | 8 +
.../om/request/util/OMMultipartUploadUtils.java | 13 +-
.../om/request/volume/OMQuotaRepairRequest.java | 11 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 11 +-
.../om/response/key/OMKeysDeleteResponse.java | 17 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 15 +-
.../OMLifecycleConfigurationDeleteResponse.java} | 42 +-
.../OMLifecycleConfigurationSetResponse.java} | 43 +-
.../OMLifecycleSaveScanStateResponse.java} | 29 +-
.../OMLifecycleSetServiceStatusResponse.java} | 16 +-
.../ozone/om/response/lifecycle}/package-info.java | 4 +-
.../snapshot/OMSnapshotMoveTableKeysResponse.java | 7 +-
.../hadoop/ozone/om/service/CompactDBUtil.java | 22 +-
.../hadoop/ozone/om/service/CompactionService.java | 12 +-
.../ozone/om/service/DirectoryDeletingService.java | 5 +-
.../ozone/om/service/KeyDeletingService.java | 15 +-
.../ozone/om/service/KeyLifecycleService.java | 2172 +++++
.../om/service/KeyLifecycleServiceMetrics.java | 196 +
.../ozone/om/service/OMRangerBGSyncService.java | 4 +-
.../om/snapshot/FSODirectoryPathResolver.java | 3 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 168 +-
.../om/snapshot/defrag/SnapshotDefragService.java | 147 +-
.../diff/delta/CompositeDeltaDiffComputer.java | 3 +
.../om/snapshot/filter/ReclaimableFilter.java | 15 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 40 +
.../apache/hadoop/ozone/om}/FaultInjectorImpl.java | 2 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 212 +
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 61 +-
...DBCheckpointServletInodeBasedXferNonLeader.java | 86 +-
.../hadoop/ozone/om/TestOMMetadataReader.java | 2 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 231 +-
.../hadoop/ozone/om/lock/TestKeyPathLock.java | 24 +-
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 150 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 279 +
.../om/request/key/TestOMKeyCommitRequest.java | 4 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 158 +-
.../key/TestOMKeysDeleteRequestWithFSO.java | 77 +-
.../TestOMLifecycleConfigurationDeleteRequest.java | 296 +
.../TestOMLifecycleConfigurationRequest.java | 153 +
.../TestOMLifecycleConfigurationSetRequest.java | 386 +
.../TestOMLifecycleSaveScanStateRequest.java | 117 +
.../s3/multipart/S3MultipartRequestTests.java | 64 +-
.../TestS3InitiateMultipartUploadRequest.java | 108 +
.../TestS3MultipartUploadAbortRequest.java | 88 +
.../TestS3MultipartUploadCommitPartRequest.java | 125 +-
.../TestS3MultipartUploadCompleteRequest.java | 79 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 6 +-
.../snapshot/TestOMSnapshotDeleteRequest.java | 49 +
.../request/volume/TestOMQuotaRepairRequest.java | 95 +
.../ozone/om/response/TestCleanupTableInfo.java | 2 +
.../om/response/key/TestOMKeyCommitResponse.java | 2 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 2 +-
.../om/response/key/TestOMKeysDeleteResponse.java | 2 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 2 +-
...TestOMLifecycleConfigurationDeleteResponse.java | 117 +
.../TestOMLifecycleConfigurationSetResponse.java | 124 +
.../TestOMLifecycleSaveScanStateResponse.java | 61 +
.../TestS3MultipartUploadCommitPartResponse.java | 2 +-
.../TestOMSnapshotMoveTableKeysResponse.java | 6 +-
.../hadoop/ozone/om/service/TestCompactDBUtil.java | 92 +
.../ozone/om/service/TestCompactionService.java | 33 +
.../ozone/om/service/TestKeyDeletingService.java | 247 +-
.../ozone/om/service/TestKeyLifecycleService.java | 3673 ++++++++
.../om/snapshot/TestFSODirectoryPathResolver.java | 2 +-
.../ozone/om/snapshot/TestSnapshotCache.java | 16 +
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 162 +-
.../om/snapshot/TestSnapshotDiffManagerMXBean.java | 4 +-
.../snapshot/defrag/TestSnapshotDefragService.java | 319 +-
.../snapshot/filter/TestReclaimableKeyFilter.java | 71 +
.../om/upgrade/OMVersionManagerTestUtils.java | 15 +
.../ozone/om/upgrade/TestOMLayoutFeature.java | 6 +-
.../protocolPB/TestOzoneManagerRequestHandler.java | 64 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 9 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 39 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 12 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 29 +-
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 8 +-
...java => TestBasicOzoneClientAdapterHeadOp.java} | 90 +-
.../hadoop/fs/ozone/TestBasicOzoneFileSystems.java | 49 +
...mHeadOp.java => TestOzoneFileSystemHeadOp.java} | 85 +-
.../recon/schema/ContainerSchemaDefinition.java | 12 +-
hadoop-ozone/recon/pom.xml | 4 -
.../hadoop/ozone/recon/ReconRestServletModule.java | 41 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 10 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 57 +-
.../ozone/recon/api/OMDBInsightEndpoint.java | 3 +-
.../ozone/recon/api/ReconGlobalMetricsService.java | 4 +-
.../ozone/recon/api/TriggerDBSyncEndpoint.java | 12 +
.../ozone/recon/api/handlers/EntityHandler.java | 122 +-
.../ozone/recon/api/handlers/FSOBucketHandler.java | 3 +-
.../recon/api/handlers/LegacyBucketHandler.java | 5 +-
.../ozone/recon/api/handlers/OBSBucketHandler.java | 8 +-
.../recon/api/types/OMDBReprocessResponse.java} | 35 +-
.../ozone/recon/chatbot/ChatbotConfigKeys.java | 35 +
.../recon/chatbot/agent/ChatHistoryBuilder.java | 175 +
.../ozone/recon/chatbot/agent/ChatbotAgent.java | 90 +-
.../ozone/recon/chatbot/api/ChatbotEndpoint.java | 63 +-
.../recon/chatbot/llm/LangChain4jDispatcher.java | 27 +
.../ozone/recon/metrics/NSSummaryMetrics.java | 91 +
.../recon/recovery/ReconOmMetadataManagerImpl.java | 9 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 6 +-
.../scm/ReconStorageContainerManagerFacade.java | 3 +-
.../recon/spi/OzoneManagerServiceProvider.java | 7 +
.../recon/spi/ReconNamespaceSummaryManager.java | 6 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 237 +-
.../impl/ReconContainerMetadataManagerImpl.java | 21 +-
.../ozone/recon/spi/impl/ReconDBProvider.java | 16 -
.../spi/impl/ReconFileMetadataManagerImpl.java | 5 +-
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 24 +-
.../recon/spi/impl/ReconRDBSnapshotProvider.java | 245 +
.../impl/StorageContainerServiceProviderImpl.java | 3 +-
.../recon/tasks/ContainerKeyMapperHelper.java | 2 +-
.../recon/tasks/DeletedKeysInsightHandler.java | 2 +-
.../recon/tasks/MultipartInfoInsightHandler.java | 219 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 3 +-
.../ozone/recon/tasks/OpenKeysInsightHandler.java | 2 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 158 +-
.../tasks/ReconTaskReInitializationEvent.java | 3 +-
.../tasks/util/ParallelTableIteratorOperation.java | 4 +-
.../ozone/recon/upgrade/ReconUpgradeAction.java | 12 +-
.../recon-tool-selection-prompt-preamble.txt | 7 +
.../webapps/recon/ozone-recon-web/package.json | 6 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 1428 +--
.../src/__tests__/Overview.test.tsx | 80 +-
.../src/__tests__/assistant/Assistant.test.tsx | 30 +-
.../src/__tests__/capacity/Capacity.test.tsx | 330 +-
.../src/__tests__/datanodes/Datanodes.test.tsx | 27 +-
.../__tests__/datanodes/DatanodesTable.test.tsx | 32 +-
.../src/__tests__/locators/locators.ts | 6 +-
.../mocks/assistantMocks/assistantServer.ts | 168 +-
.../mocks/capacityMocks/capacityServer.ts | 34 +-
.../mocks/datanodeMocks/datanodeResponseMocks.ts | 17 +-
.../mocks/datanodeMocks/datanodeServer.ts | 44 +-
.../mocks/overviewMocks/overviewResponseMocks.ts | 43 +
.../mocks/overviewMocks/overviewServer.ts | 58 +-
.../mocks/pipelineMocks/pipelinesServer.ts | 9 +-
.../src/__tests__/pipelines/Pipelines.test.tsx | 30 +-
.../src/v2/components/cards/overviewHealthCard.tsx | 5 +-
.../src/v2/components/storageBar/storageBar.tsx | 41 +-
.../ozone-recon-web/src/v2/hooks/useChat.hook.tsx | 28 +-
.../src/v2/pages/capacity/capacity.tsx | 92 +-
.../pages/capacity/components/CapacityDetail.tsx | 4 +-
.../src/v2/pages/datanodes/datanodes.tsx | 6 +-
.../src/v2/pages/overview/overview.tsx | 3 +-
.../src/v2/pages/pipelines/pipelines.tsx | 2 +-
.../ozone-recon-web/src/v2/types/chatbot.types.ts | 8 +
.../ozone/recon/OMMetadataManagerTestUtils.java | 36 +
.../apache/hadoop/ozone/recon/TestReconUtils.java | 40 +
.../recon/api/AbstractTreeNSSummaryScenario.java | 806 ++
.../ozone/recon/api/FlatNSSummaryScenario.java | 655 ++
.../ozone/recon/api/FsoNSSummaryScenario.java | 153 +
.../ozone/recon/api/LegacyNSSummaryScenario.java | 130 +
.../ozone/recon/api/NSSummaryEndpointTestBase.java | 236 +
.../ozone/recon/api/NSSummaryTestScenario.java | 163 +
.../hadoop/ozone/recon/api/NSSummaryTests.java | 195 -
.../ozone/recon/api/TestNSSummaryEndpoint.java | 389 +
.../recon/api/TestNSSummaryEndpointWithFSO.java | 1577 ----
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 1491 ----
.../api/TestNSSummaryEndpointWithOBSAndLegacy.java | 1452 ---
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 1 +
.../api/handlers/TestEntityHandlerCycleGuard.java | 196 +
.../chatbot/agent/TestChatHistoryBuilder.java | 289 +
.../agent/TestChatbotAgentExecutionPolicy.java | 62 +
.../recon/chatbot/api/TestChatbotEndpoint.java | 14 +-
.../chatbot/llm/TestLangChain4jDispatcher.java | 57 +
.../ozone/recon/chatbot/llm/TestLlmRouting.java | 8 +
.../ozone/recon/metrics/TestNSSummaryMetrics.java | 60 +
.../impl/TestOzoneManagerServiceProviderImpl.java | 321 +-
.../TestReconContainerMetadataManagerImpl.java | 22 +
.../spi/impl/TestReconRDBSnapshotProvider.java | 223 +
.../ozone/recon/tasks/TestFileSizeCountTask.java | 23 +-
.../ozone/recon/tasks/TestOmTableInsightTask.java | 129 +
.../recon/tasks/TestReconTaskControllerImpl.java | 160 +-
hadoop-ozone/s3gateway/pom.xml | 5 -
.../org/apache/hadoop/ozone/audit/S3GAction.java | 7 +-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 37 +-
.../ozone/s3/commontypes/RequestParameters.java | 14 +
.../ozone/s3/endpoint/BucketCrudHandler.java | 3 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 23 +-
.../ozone/s3/endpoint/BucketLifecycleHandler.java | 170 +
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 106 +-
.../s3/endpoint/GetObjectAttributesResponse.java | 206 +
.../ozone/s3/endpoint/ObjectAttributesHandler.java | 173 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 138 +-
...onHandler.java => ObjectGetTorrentHandler.java} | 15 +-
...tBucketLifecycleConfigurationUnmarshaller.java} | 56 +-
.../s3/endpoint/S3LifecycleConfiguration.java | 558 ++
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 7 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 22 +
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 14 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 19 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 29 +
.../hadoop/ozone/client/OzoneBucketStub.java | 68 +
.../hadoop/ozone/client/OzoneVolumeStub.java | 2 +
.../ozone/s3/TestVirtualHostStyleFilter.java | 204 +-
.../ozone/s3/endpoint/EndpointTestUtils.java | 14 +
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 37 +
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 60 +-
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 26 +
.../s3/endpoint/TestMultipartUploadComplete.java | 22 +
.../ozone/s3/endpoint/TestObjectAttributesGet.java | 176 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 67 +-
...tGetLocation.java => TestObjectGetTorrent.java} | 18 +-
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 99 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 87 +
.../TestS3LifecycleConfigurationDelete.java | 96 +
.../endpoint/TestS3LifecycleConfigurationGet.java | 205 +
.../endpoint/TestS3LifecycleConfigurationPut.java | 665 ++
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 20 +
hadoop-ozone/tools/pom.xml | 15 +
.../hadoop/ozone/local/LocalOzoneCluster.java | 228 +-
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 64 +-
.../hadoop/ozone/local/TestLocalOzoneCluster.java | 109 +
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 120 +-
hadoop-ozone/vapor/pom.xml | 11 +
pom.xml | 217 +-
763 files changed, 60489 insertions(+), 12812 deletions(-)
create mode 100644 hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/CliOptionStyleProcessor.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientShortCircuit.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/DomainPeer.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/DomainSocketFactory.java
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/LocalChunkInputStream.java
create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientGrpcChannel.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/{SkipTracing.java => InternalSpanKind.java} (86%)
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestHostAndPort.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingAnnotations.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingInitModes.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestCompositeKey.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumMultiBuffer.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/utils/TestBufferUtils.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/Receiver.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DatanodeStorageMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/VolumeReplicationThreadPools.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTaskDnsRefresh.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestDatanodeStorageMetrics.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestReconcileChunksPerBlockHoleBcsId.java
create mode 100644 hadoop-hdds/docs/content/design/lifecycle-task-resume.md
copy hadoop-hdds/docs/content/design/{dn-merge-rocksdb.md => s3-object-lifecycle-management.md} (59%)
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-current-flow.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-getblock-flow.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-hbase-benchmark.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-mmap-cache-benchmark.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-target-flow.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read-unix-domain-socket.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read.md
create mode 100644 hadoop-hdds/docs/content/feature/Lifecycle.md
create mode 100644 hadoop-hdds/docs/content/feature/Lifecycle.zh.md
create mode 100644 hadoop-hdds/docs/content/feature/Short-Circuit-Read.md
create mode 100644 hadoop-hdds/docs/content/feature/Short-Circuit-Read.zh.md
delete mode 100644 hadoop-hdds/docs/content/interface/pvtest-webui.png
create mode 100644 hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCodecRegistryTcclIsolation.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/DnsNames.java
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/utils/TestDnsNames.java
create mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/LatestVersionedKWayMergeIterator.java
create mode 100644 hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestLatestVersionedKWayMergeIterator.java
create mode 100644 hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestLatestVersionedKWayMergeIteratorOverSst.java
create mode 100644 hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/TestRawSstFileRecords.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportJob.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java
copy hadoop-hdds/{client/src/test/java/org/apache/hadoop/hdds/scm/storage => server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export}/package-info.java (87%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java
copy hadoop-ozone/{cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/quota/QuotaStatus.java => cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleResumeSubCommand.java} (55%)
create mode 100644 hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java
copy hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/{upgrade/UpgradeCommands.java => om/LifecycleSubCommand.java} (68%)
create mode 100644 hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java
create mode 100644 hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommandUtil.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneLifecycleConfiguration.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCAbortIncompleteMultipartUpload.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCAction.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/OMLCUtils.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCAbortIncompleteMultipartUpload.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifeCycleConfiguration.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleScanState.java
delete mode 100644 hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Datanode Chunk Read_Write Dashboard.json
create mode 100644 hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - DataNode Overview.json
create mode 100644 hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Disk Balancer.json
copy hadoop-ozone/dist/src/main/compose/{ozonesecure/diskbalancer.yaml => ozone/short-circuit.yaml} (80%)
copy hadoop-ozone/dist/src/main/compose/ozone/{test-failures1.sh => test-short-circuit.sh} (79%)
copy hadoop-ozone/dist/src/main/compose/ozonesecure-ha/{test-s3g-virtual-host.sh => test-om-lifecycle.sh} (86%)
mode change 100755 => 100644
create mode 100644 hadoop-ozone/dist/src/main/smoketest/lifecycle/om-lifecycle.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/s3/bucketlifecycle.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/short-circuit/short-circuit.robot
copy hadoop-ozone/dist/src/{main/k8s/examples/getting-started/LICENSE.header => shell/conf/shell-logging.properties} (98%)
copy hadoop-ozone/dist/src/test/shell/{ozone-functions_test_helper.bash => ozone_bootstrap.bats} (61%)
mode change 100755 => 100644
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemDataStreamEnablement.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRackAwarePlacement.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{TestDataUtil.java => DataTestUtil.java} (98%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestDomainSocketFactory.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/{TestHelper.java => OzoneTestHelper.java} (99%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeStorageMetricsIntegration.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMMXBeanTests.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMInstallSnapshotDuringBootstrapping.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotCheckpointDbContent.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java
create mode 100644 hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/FixedHostMapping.java
create mode 100644 hadoop-ozone/multitenancy-ranger/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestRangerClientMultiTenantAccessControllerMockClient.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationDeleteRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSetServiceStatusRequest.java
copy {hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle}/package-info.java (86%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{volume/OMVolumeSetQuotaResponse.java => lifecycle/OMLifecycleConfigurationDeleteResponse.java} (59%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{volume/OMVolumeSetQuotaResponse.java => lifecycle/OMLifecycleConfigurationSetResponse.java} (54%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{util/OMEchoRPCWriteResponse.java => lifecycle/OMLifecycleSaveScanStateResponse.java} (56%)
copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{util/OMEchoRPCWriteResponse.java => lifecycle/OMLifecycleSetServiceStatusResponse.java} (75%)
copy {hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage => hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle}/package-info.java (86%)
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleServiceMetrics.java
copy hadoop-ozone/{integration-test/src/test/java/org/apache/hadoop/utils => ozone-manager/src/test/java/org/apache/hadoop/ozone/om}/FaultInjectorImpl.java (98%)
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationDeleteRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMQuotaRepairRequest.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleConfigurationDeleteResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleConfigurationSetResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleSaveScanStateResponse.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
copy hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/{TestBasicRootedOzoneClientAdapterHeadOp.java => TestBasicOzoneClientAdapterHeadOp.java} (57%)
copy hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/{TestRootedOzoneFileSystemHeadOp.java => TestOzoneFileSystemHeadOp.java} (60%)
copy hadoop-ozone/{common/src/main/java/org/apache/hadoop/ozone/om/helpers/ErrorInfo.java => recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OMDBReprocessResponse.java} (63%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/chatbot/agent/ChatHistoryBuilder.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/NSSummaryMetrics.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRDBSnapshotProvider.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FsoNSSummaryScenario.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/LegacyNSSummaryScenario.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpointTestBase.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTestScenario.java
delete mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTests.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java
delete mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java
delete mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java
delete mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/handlers/TestEntityHandlerCycleGuard.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/chatbot/agent/TestChatHistoryBuilder.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/metrics/TestNSSummaryMetrics.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRDBSnapshotProvider.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketLifecycleHandler.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/GetObjectAttributesResponse.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/{BucketGetLocationHandler.java => ObjectGetTorrentHandler.java} (73%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/{MessageUnmarshaller.java => PutBucketLifecycleConfigurationUnmarshaller.java} (52%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java
copy hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/{TestBucketGetLocation.java => TestObjectGetTorrent.java} (70%)
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationDelete.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org
(ozone) 01/01: Merge branch 'master' into HDDS-14496-zdu
Posted by er...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a commit to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 71a27e3fc1f7feebf3fd1ffe8fca051055bdd1b3
Merge: 25495e1a9f5 50978e3ad7c
Author: Ethan Rose <er...@apache.org>
AuthorDate: Tue Aug 25 13:40:49 2026 -0400
Merge branch 'master' into HDDS-14496-zdu
* master: (196 commits)
HDDS-15422. Stream read seek should not close stream (#10415)
HDDS-16089. Improve MPU table evaluation efficiency in KeyLifeCycleService (#10949)
HDDS-16169. Add old updateContainerState to handle upgrade path. (#11012)
HDDS-16158. Fix sendDeleteKeysRequestAndClearList Authorization and FSO Trash Handling (#10996)
HDDS-15911. Recon AI Assistant: add conversation memory for follow-up questions. (#10808)
HDDS-15356. Make multi-buffer chunk checksum allocation-free (#10350)
HDDS-15895. Make ServerNotLeaderException suggested leader flow IPv6-safe (#10807)
HDDS-16118. Release snapshot DB handles before KeyDeletingService submits OM requests (#10987)
HDDS-16125. Copy bucket snapshot in quota repair response to fix cache aliasing (#11004)
HDDS-8082. Check for unnecessary newlines in config defaults (#10840)
HDDS-15826. Recon: add cycle guard to NSSummary /du tree walks (#10723)
HDDS-11620. Log SCM finalization completion (#10963)
HDDS-16117. Failed FSO multipart complete leaks bucket namespace quota (#10984)
HDDS-15961. Resolve linked bucket source properties consistently (#10859)
HDDS-16072. Implement GetObjectAttributes: ETag, ObjectSize, StorageClass, and ObjectParts count (#10930).
HDDS-16114. Avoid misleading Ozone mount warning in acceptance test results (#10979)
HDDS-15761. Create test for OMMXBean (#10960)
HDDS-15542. Closed-container reconciliation advances BCSID past a hole, masking missing chunks (#10496)
HDDS-15776. Make S3 Gateway VirtualHostStyleFilter IPv6-safe (#10809)
HDDS-15775. Support IPv6 literal authorities in OzoneFS URI parsing (#10933)
...
Files with conflicts or requiring updates:
---
hadoop-hdds/common/src/main/resources/ozone-default.xml
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/upgrade/TestScmVersionManager.java
hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialConstraintUpgradeAction.java
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/UnhealthyContainerReplicaMismatchAction.java
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTaskDnsRefresh.java
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/TestDecommissionAndMaintenance.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestFinalizeBlock.java
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupInvalidApparentVersion.java
hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationDeleteRequest.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManagerTestUtils.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java
.asf.yaml | 1 -
.github/workflows/asf-allowlist-check.yaml | 2 +-
.github/workflows/build-ratis.yml | 6 +-
.github/workflows/check.yml | 4 +-
.github/workflows/ci.yml | 12 +-
.github/workflows/close-stale-prs.yaml | 3 +-
.github/workflows/generate-config-doc.yml | 4 +-
.github/workflows/intermittent-test-check.yml | 10 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/populate-cache.yml | 6 +-
.github/workflows/pull-request.yml | 2 +-
.github/workflows/repeat-acceptance.yml | 8 +-
.github/workflows/update-ozone-site-config-doc.yml | 57 +-
.github/workflows/zizmor.yml | 6 +-
AGENTS.md | 2 +-
CONTRIBUTING.md | 2 +-
THREAT_MODEL.md | 16 +-
.../ozone/annotations/CliOptionStyleProcessor.java | 123 +
hadoop-hdds/cli-common/pom.xml | 11 +
hadoop-hdds/client/pom.xml | 8 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 68 +
.../hadoop/hdds/scm/XceiverClientCreator.java | 37 +-
.../hadoop/hdds/scm/XceiverClientFactory.java | 19 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 13 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 103 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 3 +-
.../hadoop/hdds/scm/XceiverClientShortCircuit.java | 623 ++
.../hdds/scm/storage/BlockDataStreamOutput.java | 28 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 149 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 16 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +-
.../apache/hadoop/hdds/scm/storage/DomainPeer.java | 111 +
.../hdds/scm/storage/DomainSocketFactory.java | 276 +
.../hdds/scm/storage/ECBlockOutputStream.java | 25 +-
.../hdds/scm/storage/LocalChunkInputStream.java | 116 +
.../hdds/scm/storage/MultipartInputStream.java | 12 +-
.../hdds/scm/storage/StreamBlockInputStream.java | 186 +-
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 17 +
.../hdds/scm/TestXceiverClientGrpcChannel.java | 68 +
.../hdds/scm/client/TestHddsClientUtils.java | 37 +
.../hdds/scm/storage/MockDatanodePipeline.java | 29 +-
.../scm/storage/TestBlockDataStreamOutput.java | 22 +-
.../storage/TestBlockOutputStreamCorrectness.java | 52 +-
hadoop-hdds/common/src/main/conf/ozone-env.sh | 8 +-
.../java/org/apache/hadoop/hdds/HDDSVersion.java | 1 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 4 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 85 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 74 +-
.../hdds/ratis/ServerNotLeaderException.java | 5 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 3 +
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 14 +
.../balancer/ContainerBalancerConfiguration.java | 17 +-
.../apache/hadoop/hdds/scm/net/HostAndPort.java | 23 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 1 +
.../ContainerCommandResponseBuilders.java | 20 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 65 +-
.../hadoop/hdds/tracing/GrpcServerInterceptor.java | 4 +-
.../hadoop/hdds/tracing/InternalSpanKind.java} | 45 +-
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 9 +-
.../apache/hadoop/hdds/tracing/TracingConfig.java | 16 +
.../apache/hadoop/hdds/tracing/TracingUtil.java | 245 +-
.../org/apache/hadoop/hdds/utils/CompositeKey.java | 109 +-
.../apache/hadoop/hdds/utils/SimpleStriped.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../org/apache/hadoop/ozone/common/Checksum.java | 200 +-
.../apache/hadoop/ozone/common/ChecksumCache.java | 117 +-
.../hadoop/ozone/common/utils/BufferUtils.java | 23 +
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 24 +-
.../org/apache/hadoop/ozone/util/UUIDUtil.java | 9 +
.../common/src/main/resources/ozone-default.xml | 114 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 90 +
.../hadoop/hdds/protocol/TestDatanodeDetails.java | 37 +
...TestServerNotLeaderExceptionMessageParsing.java | 22 +
.../hadoop/hdds/scm/net/TestHostAndPort.java | 62 +
.../hadoop/hdds/tracing/TestTraceAllMethod.java | 20 +
.../hdds/tracing/TestTracingAnnotations.java | 88 +
.../hadoop/hdds/tracing/TestTracingInitModes.java | 193 +
.../hadoop/hdds/tracing/TestTracingUtil.java | 77 +-
.../apache/hadoop/hdds/utils/TestCompositeKey.java | 108 +
.../hadoop/hdds/utils/TestProtobufUtils.java | 36 +
.../hadoop/hdds/utils/TestSimpleStriped.java | 3 +-
.../apache/hadoop/ozone/common/TestChecksum.java | 17 +
.../hadoop/ozone/common/TestChecksumCache.java | 73 +-
.../ozone/common/TestChecksumMultiBuffer.java | 97 +
.../hadoop/ozone/common/utils/TestBufferUtils.java | 81 +
.../ozone/container/ContainerTestHelper.java | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 28 +-
.../org/apache/hadoop/ozone/audit/DNAction.java | 1 +
.../checksum/ContainerChecksumTreeManager.java | 12 +-
.../container/checksum/ContainerDiffReport.java | 13 +
.../container/common/helpers/ContainerMetrics.java | 49 +-
.../container/common/helpers/ContainerUtils.java | 26 +
.../ozone/container/common/impl/ContainerSet.java | 95 +-
.../container/common/impl/HddsDispatcher.java | 20 +-
.../common/interfaces/ContainerDispatcher.java | 6 +-
.../ozone/container/common/interfaces/Handler.java | 27 +-
.../common/statemachine/DatanodeStateMachine.java | 7 +
.../common/statemachine/EndpointStateMachine.java | 10 +-
.../common/statemachine/SCMConnectionManager.java | 98 +-
.../states/endpoint/HeartbeatEndpointTask.java | 32 +
.../common/transport/server/Receiver.java | 353 +
.../server/XceiverServerDomainSocket.java | 324 +
.../common/transport/server/XceiverServerGrpc.java | 57 +-
.../common/transport/server/XceiverServerSpi.java | 2 +
.../server/ratis/ContainerStateMachine.java | 20 +-
.../transport/server/ratis/XceiverServerRatis.java | 9 +-
.../common/volume/DatanodeStorageMetrics.java | 123 +
.../container/common/volume/MutableVolumeSet.java | 37 +
.../diskbalancer/DiskBalancerService.java | 17 +-
.../ECReconstructionCoordinator.java | 3 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 246 +-
.../container/keyvalue/helpers/BlockUtils.java | 9 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 24 +
.../keyvalue/impl/ChunkManagerDispatcher.java | 15 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 30 +-
.../keyvalue/impl/KeyValueStreamDataChannel.java | 104 +-
.../keyvalue/impl/StreamDataChannelBase.java | 16 +
.../keyvalue/interfaces/BlockManager.java | 10 +
.../keyvalue/interfaces/ChunkManager.java | 22 +-
.../StaleRecoveringContainerScrubbingService.java | 21 +-
.../ozone/container/metadata/DatanodeTable.java | 12 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 82 +-
.../container/replication/ReplicationServer.java | 73 +-
.../replication/ReplicationSupervisor.java | 183 +-
.../replication/ReplicationSupervisorMetrics.java | 5 +-
.../replication/VolumeReplicationThreadPools.java | 166 +
.../ContainerTableSchemaFinalizeAction.java | 2 +-
.../TestSchemaOneBackwardsCompatibility.java | 19 +-
...stStaleRecoveringContainerScrubbingService.java | 50 +
.../container/common/impl/TestHddsDispatcher.java | 75 +
.../statemachine/TestSCMConnectionManager.java | 91 +
.../TestHeartbeatEndpointTaskDnsRefresh.java | 136 +
.../common/volume/TestDatanodeStorageMetrics.java | 162 +
.../container/common/volume/TestVolumeSet.java | 27 +
.../diskbalancer/TestDiskBalancerService.java | 43 +
.../container/keyvalue/TestKeyValueHandler.java | 2 +-
.../TestReconcileChunksPerBlockHoleBcsId.java | 522 ++
.../keyvalue/impl/TestFilePerBlockStrategy.java | 10 +-
.../impl/TestKeyValueStreamDataChannel.java | 139 +-
.../replication/TestReplicationConfig.java | 30 +
.../replication/TestReplicationSupervisor.java | 473 +-
.../upgrade/TestDatanodeVersionManager.java | 2 +-
hadoop-hdds/docs/content/design/diskbalancer.md | 12 +-
.../docs/content/design/lifecycle-task-resume.md | 85 +
.../design/s3-object-lifecycle-management.md | 29 +
.../design/short-circuit-read-current-flow.png | Bin 0 -> 33740 bytes
.../design/short-circuit-read-getblock-flow.png | Bin 0 -> 66181 bytes
.../design/short-circuit-read-hbase-benchmark.png | Bin 0 -> 34029 bytes
.../short-circuit-read-mmap-cache-benchmark.png | Bin 0 -> 28688 bytes
.../design/short-circuit-read-target-flow.png | Bin 0 -> 39362 bytes
.../short-circuit-read-unix-domain-socket.png | Bin 0 -> 82958 bytes
.../docs/content/design/short-circuit-read.md | 175 +
hadoop-hdds/docs/content/feature/Decommission.md | 9 +-
hadoop-hdds/docs/content/feature/DiskBalancer.md | 37 +-
.../docs/content/feature/DiskBalancer.zh.md | 34 +-
hadoop-hdds/docs/content/feature/Lifecycle.md | 397 +
hadoop-hdds/docs/content/feature/Lifecycle.zh.md | 396 +
.../docs/content/feature/Reconfigurability.md | 3 +-
.../docs/content/feature/Short-Circuit-Read.md | 90 +
.../docs/content/feature/Short-Circuit-Read.zh.md | 88 +
.../docs/content/interface/pvtest-webui.png | Bin 13830 -> 0 bytes
hadoop-hdds/docs/content/tools/_index.md | 2 +
.../apache/ozone/erasurecode/CodecRegistry.java | 3 +-
.../TestCodecRegistryTcclIsolation.java | 52 +
.../hadoop/hdds/scm/client/ScmTopologyClient.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 4 +-
.../scm/proxy/SCMFailoverProxyProviderBase.java | 95 +-
.../authority/profile/DefaultProfile.java | 4 +-
.../client/DefaultCertificateClient.java | 11 +
.../certificate/utils/CertificateSignRequest.java | 46 +-
.../security/x509/certificate/utils/DnsNames.java | 107 +
.../certificate/utils/SelfSignedCertificate.java | 59 +-
.../server/OzoneProtocolMessageDispatcher.java | 3 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 37 +-
.../hadoop/hdds/utils/RDBSnapshotProvider.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 28 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 9 +
.../TestSCMFailoverProxyProviderRefreshWired.java | 167 +-
.../certificate/authority/TestDefaultCAServer.java | 45 +
.../certificate/authority/TestDefaultProfile.java | 42 +
.../client/TestDefaultCertificateClient.java | 50 +
.../utils/TestCertificateSignRequest.java | 125 +
.../x509/certificate/utils/TestDnsNames.java | 93 +
.../certificate/utils/TestRootCertificate.java | 47 +
.../hadoop/hdds/utils/db/InMemoryTestTable.java | 5 +
.../hadoop/hdds/utils/db/TestTypedTable.java | 48 +
.../src/main/proto/DatanodeClientProtocol.proto | 16 +
.../dev-support/findbugsExcludeFile.xml | 9 +
hadoop-hdds/rocks-native/pom.xml | 6 +-
.../utils/db/LatestVersionedKWayMergeIterator.java | 504 ++
.../db/TestLatestVersionedKWayMergeIterator.java | 253 +
...estLatestVersionedKWayMergeIteratorOverSst.java | 204 +
.../hdds/utils/db/TestRawSstFileRecords.java | 121 +
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 48 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 55 +-
hadoop-hdds/server-scm/pom.xml | 6 +
.../hdds/scm/container/ContainerStateManager.java | 6 +
.../scm/container/ContainerStateManagerImpl.java | 30 +
.../scm/container/balancer/ContainerBalancer.java | 158 +
.../container/balancer/ContainerBalancerTask.java | 16 +-
.../scm/container/export/ExportFileManager.java | 234 +
.../hdds/scm/container/export/ExportJob.java | 74 +
.../hdds/scm/container/export/ExportScope.java | 68 +
.../hdds/scm/container/export/package-info.java} | 37 +-
.../SCMContainerPlacementRackScatter.java | 49 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 2 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 31 +-
.../ha/invoker/ContainerStateManagerInvoker.java | 25 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 5 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 68 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 2 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 263 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 55 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 9 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 26 +
.../container/balancer/TestContainerBalancer.java | 107 +-
.../container/export/TestExportFileManager.java | 156 +
.../TestSCMContainerPlacementRackScatter.java | 34 +
.../hadoop/hdds/scm/ha/TestSCMRatisServerImpl.java | 62 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 76 +
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 187 +
.../scm/pipeline/TestRatisPipelineProvider.java | 90 +-
.../scm/server/upgrade/TestScmVersionManager.java | 2 +-
hadoop-ozone/cli-admin/pom.xml | 15 +
.../scm/cli/ContainerBalancerStartSubcommand.java | 2 -
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 49 +-
.../datanode/AbstractDiskBalancerSubCommand.java | 162 +-
.../hdds/scm/cli/datanode/DatanodeParameters.java | 3 +-
.../scm/cli/datanode/DiskBalancerCommands.java | 31 +-
.../cli/datanode/DiskBalancerCommonOptions.java | 12 +
.../cli/datanode/DiskBalancerReportSubcommand.java | 37 +-
.../cli/datanode/DiskBalancerStatusSubcommand.java | 48 +-
.../cli/datanode/DiskBalancerSubCommandUtil.java | 139 +-
.../ozone/admin/nssummary/NSSummaryAdmin.java | 31 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 3 +-
.../ozone/admin/om/LifecycleResumeSubCommand.java | 72 +
.../ozone/admin/om/LifecycleStatusSubCommand.java | 88 +
.../hadoop/ozone/admin/om/LifecycleSubCommand.java | 51 +-
.../ozone/admin/om/LifecycleSuspendSubCommand.java | 75 +
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 3 +-
.../admin/scm/GetScmRatisRolesSubcommand.java | 51 +-
.../datanode/TestContainerBalancerSubCommand.java | 4 +-
.../datanode/TestDiskBalancerSubCommandUtil.java | 135 +
.../cli/datanode/TestDiskBalancerSubCommands.java | 345 +-
.../ozone/scm/TestGetScmRatisRolesSubcommand.java | 65 +
hadoop-ozone/cli-debug/pom.xml | 11 +
.../ozone/debug/om/ContainerToKeyMapping.java | 82 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 3 +-
.../ozone/debug/om/TestContainerToKeyMapping.java | 59 +
hadoop-ozone/cli-repair/pom.xml | 11 +
.../schemaupgrade/UpgradeContainerSchema.java | 3 +-
.../ozone/repair/ldb/RocksDBManualCompaction.java | 15 +-
.../apache/hadoop/ozone/repair/om/CompactOMDB.java | 14 +-
.../hadoop/ozone/repair/om/FSORepairTool.java | 25 +-
.../hadoop/ozone/repair/ldb/TestLdbRepair.java | 2 +-
hadoop-ozone/cli-shell/pom.xml | 15 +
.../java/org/apache/hadoop/ozone/shell/Shell.java | 3 +-
.../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 4 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 7 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 34 +
.../hadoop/ozone/client/OzoneClientFactory.java | 5 +-
.../ozone/client/OzoneLifecycleConfiguration.java | 230 +
.../ozone/client/protocol/ClientProtocol.java | 45 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 60 +-
.../ozone/client/MockXceiverClientFactory.java | 9 +-
.../client/checksum/FileChecksumBenchmark.java | 12 +
hadoop-ozone/common/pom.xml | 4 +
.../apache/hadoop/fs/ozone/OzoneTrashPolicy.java | 2 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 18 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 53 +
.../java/org/apache/hadoop/ozone/om/OmConfig.java | 16 +
.../hadoop/ozone/om/exceptions/OMException.java | 3 +
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 26 +
.../OmLCAbortIncompleteMultipartUpload.java | 139 +
.../apache/hadoop/ozone/om/helpers/OmLCAction.java | 59 +
.../hadoop/ozone/om/helpers/OmLCExpiration.java | 215 +
.../apache/hadoop/ozone/om/helpers/OmLCFilter.java | 235 +
.../apache/hadoop/ozone/om/helpers/OmLCRule.java | 411 +
.../ozone/om/helpers/OmLifecycleConfiguration.java | 334 +
.../om/helpers/OmLifecycleRuleAndOperator.java | 214 +
.../ozone/om/helpers/OmLifecycleScanState.java | 255 +
.../hadoop/ozone/om/helpers/OmLifecycleUtils.java | 131 +
.../ozone/om/helpers/OmMultipartKeyInfo.java | 30 +-
.../ozone/om/helpers/OmMultipartPartInfo.java | 32 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 72 +
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 3 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 65 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 116 +-
.../ozone/snapshot/SnapshotDiffResponse.java | 28 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 19 +
.../apache/hadoop/ozone/om/helpers/OMLCUtils.java | 131 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 71 +
.../TestOmLCAbortIncompleteMultipartUpload.java | 171 +
.../ozone/om/helpers/TestOmLCExpiration.java | 198 +
.../hadoop/ozone/om/helpers/TestOmLCFilter.java | 159 +
.../hadoop/ozone/om/helpers/TestOmLCRule.java | 459 +
.../om/helpers/TestOmLifeCycleConfiguration.java | 266 +
.../om/helpers/TestOmLifecycleRuleAndOperator.java | 139 +
.../ozone/om/helpers/TestOmLifecycleScanState.java | 79 +
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 22 +-
.../ozone/snapshot/TestSnapshotDiffResponse.java | 40 +-
.../dist/dev-support/bin/dist-layout-stitching | 23 +-
.../Datanode Chunk Read_Write Dashboard.json | 1601 ----
.../dashboards/Ozone - DataNode Overview.json | 9417 ++++++++++++++++++++
.../grafana/dashboards/Ozone - Disk Balancer.json | 1096 +++
.../dist/src/main/compose/ozone-ha/docker-config | 26 +
.../dist/src/main/compose/ozone/docker-config | 14 +-
.../dist/src/main/compose/ozone/short-circuit.yaml | 25 +
.../src/main/compose/ozone/test-short-circuit.sh | 36 +
.../src/main/compose/ozonesecure-ha/docker-config | 6 +
.../compose/ozonesecure-ha/test-om-lifecycle.sh | 38 +
.../compose/ozonesecure-ha/test-repair-tools.sh | 2 +-
.../src/main/compose/ozonesecure/docker-config | 26 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 4 +-
.../src/main/smoketest/admincli/container.robot | 14 +-
.../src/main/smoketest/freon/read-write-key.robot | 7 +
.../main/smoketest/lifecycle/om-lifecycle.robot | 119 +
.../dist/src/main/smoketest/s3/bucketcreate.robot | 2 +-
.../src/main/smoketest/s3/bucketlifecycle.robot | 63 +
.../dist/src/main/smoketest/s3/objectputget.robot | 10 +-
.../smoketest/short-circuit/short-circuit.robot | 55 +
hadoop-ozone/dist/src/main/smoketest/testlib.sh | 3 +-
.../dist/src/shell/conf/shell-logging.properties | 17 +
hadoop-ozone/dist/src/shell/ozone/ozone | 8 +-
.../dist/src/shell/ozone/ozone-functions.sh | 55 +-
.../dist/src/test/shell/ozone_bootstrap.bats | 53 +-
hadoop-ozone/freon/pom.xml | 11 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 34 +
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 3 +-
.../apache/ozone/fs/http/server/HttpFSServer.java | 2 +-
hadoop-ozone/iceberg/pom.xml | 20 +-
hadoop-ozone/insight/pom.xml | 17 +-
.../ozone/insight/BaseInsightSubCommand.java | 35 +-
.../ozone/insight/TestBaseInsightSubCommand.java | 15 +
.../AbstractTestStorageDistributionEndpoint.java | 2 +
.../hadoop/ozone/recon/ReconEndpointTestUtil.java | 25 +-
.../ozone/recon/TestNSSummaryMemoryLeak.java | 4 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 14 +-
.../TestReconInsightsForDeletedDirectories.java | 8 +-
.../ozone/recon/TestReconWithOzoneManager.java | 116 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 4 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 9 -
hadoop-ozone/integration-test-s3/pom.xml | 12 +-
.../hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java | 11 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 443 +-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 584 ++
hadoop-ozone/integration-test/pom.xml | 16 +-
.../hadoop/conf/ConfigurationFieldsTests.java | 1 -
.../fs/ozone/AbstractOzoneFileSystemTest.java | 18 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 38 +-
.../AbstractRootedOzoneFileSystemTestWithFSO.java | 3 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 14 +-
.../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 4 +-
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 6 +-
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 4 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 149 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 8 +-
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 2 +-
.../TestOzoneFileSystemDataStreamEnablement.java | 432 +
.../fs/ozone/TestOzoneFileSystemMetrics.java | 4 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 4 +-
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 4 +-
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 4 +-
...oneFileSystemWithStreamingDisabledDatanode.java | 4 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 4 +-
.../hadoop/hdds/TestXceiverServerDomainSocket.java | 793 ++
.../apache/hadoop/hdds/scm/TestCloseContainer.java | 12 +-
.../hadoop/hdds/scm/TestContainerOperations.java | 2 +-
.../hadoop/hdds/scm/TestFailoverWithSCMHA.java | 1 +
.../hadoop/hdds/scm/TestRackAwarePlacement.java | 438 +
.../TestSCMFollowerCatchupWithContainerReport.java | 4 +-
.../hdds/scm/TestStorageContainerManager.java | 12 +-
...estStorageContainerManagerHAWithAllRunning.java | 4 +-
.../apache/hadoop/hdds/scm/TestWatchForCommit.java | 12 +-
.../hadoop/hdds/scm/TestXceiverClientManager.java | 6 +-
.../hdds/scm/TestXceiverClientManagerSC.java | 109 +
.../TestPendingContainerTrackerIntegration.java | 4 +-
.../metrics/TestSCMContainerManagerMetrics.java | 4 +-
.../TestReplicationManagerIntegration.java | 12 +-
.../scm/node/TestDecommissionAndMaintenance.java | 11 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 8 +-
.../hdds/scm/safemode/TestSafeModeSCMHA.java | 4 +-
.../TestDNDataDistributionFinalization.java | 3 +-
.../TestScmDataDistributionFinalization.java | 8 +-
.../ozone/{TestDataUtil.java => DataTestUtil.java} | 6 +-
.../ozone/TestContainerBalancerOperations.java | 24 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 66 +-
.../apache/hadoop/ozone/TestOMSortDatanodes.java | 52 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 58 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 1013 +--
.../ozone/admin/om/lease/TestLeaseRecoverer.java | 4 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 267 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../client/rpc/TestBlockDataStreamOutput.java | 63 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 6 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 12 +-
...stClientRetryContainerStateMachineFailures.java | 3 -
.../ozone/client/rpc/TestClientRetryTimeout.java | 50 +-
.../rpc/TestCloseContainerHandlingByClient.java | 8 +-
.../rpc/TestContainerReplicationEndToEnd.java | 4 +-
.../client/rpc/TestContainerStateMachine.java | 4 +-
.../rpc/TestContainerStateMachineFailures.java | 28 +-
.../rpc/TestContainerStateMachineStream.java | 34 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 4 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 8 +-
.../client/rpc/TestFailureHandlingByClient.java | 12 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 6 +-
.../client/rpc/TestHybridPipelineOnDatanode.java | 6 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 6 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 8 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 154 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 8 +-
.../rpc/TestOzoneClientRetriesOnExceptions.java | 10 +-
.../TestOzoneRpcClientWithKeyLatestVersion.java | 4 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 4 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 6 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 10 +-
.../ozone/client/rpc/read/InputStreamTests.java | 26 +-
.../client/rpc/read/TestChunkInputStream.java | 16 +-
.../client/rpc/read/TestDomainSocketFactory.java | 81 +
.../ozone/client/rpc/read/TestKeyInputStream.java | 6 +-
.../client/rpc/read/TestLocalChunkInputStream.java | 163 +
.../rpc/read/TestStreamBlockInputStream.java | 56 +-
.../ozone/client/rpc/read/TestStreamRead.java | 1 +
.../rpc/read/TestStreamReadDatanodeFailover.java | 6 +-
.../{TestHelper.java => OzoneTestHelper.java} | 6 +-
.../ozone/container/TestContainerReplication.java | 112 +-
.../container/TestContainerReportHandling.java | 12 +-
.../commandhandler/TestBlockDeletion.java | 6 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
.../commandhandler/TestFinalizeBlock.java | 3 +-
.../dn/TestDatanodeStorageMetricsIntegration.java | 134 +
.../TestContainerCommandReconciliation.java | 15 +-
...estDuplicateContainerDirScannerIntegration.java | 6 +-
.../scanner/ContainerScannerIntegrationTests.java | 6 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 441 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 4 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 4 +-
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 4 +-
.../ozone/local/TestLocalOzoneClusterRuntime.java | 34 +-
.../apache/hadoop/ozone/om/BucketForTesting.java | 4 +-
.../org/apache/hadoop/ozone/om/OMMXBeanTests.java | 88 +
.../ozone/om/TestBucketLayoutWithOlderClient.java | 8 +-
.../apache/hadoop/ozone/om/TestBucketOwner.java | 2 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 12 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 4 +-
.../org/apache/hadoop/ozone/om/TestListKeys.java | 8 +-
.../hadoop/ozone/om/TestListKeysWithFSO.java | 6 +-
.../org/apache/hadoop/ozone/om/TestListStatus.java | 4 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 8 +-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 18 +-
.../TestOMInstallSnapshotDuringBootstrapping.java | 207 +
.../ozone/om/TestOMRatisSnapshotTransfer.java | 2 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 193 +-
.../ozone/om/TestOMStartupWithBucketLayout.java | 14 +-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 4 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 8 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 4 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 30 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 18 +-
.../ozone/om/TestOmContainerLocationCache.java | 4 +
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 20 +-
.../om/TestOmStartupInvalidApparentVersion.java | 4 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 6 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 10 +-
.../ozone/om/service/TestBlockDeletionService.java | 6 +-
.../TestDirectoryDeletingServiceWithFSO.java | 4 +-
.../ozone/om/service/TestRootedDDSWithFSO.java | 13 +-
...TestSnapshotDeletingServiceIntegrationTest.java | 38 +-
.../hadoop/ozone/om/snapshot/OmSnapshotTests.java | 53 +-
.../TestOmSnapshotCheckpointDbContent.java | 511 ++
.../om/snapshot/TestOmSnapshotFileSystem.java | 6 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 6 +-
.../snapshot/TestSnapshotBackgroundServices.java | 2 +-
.../TestSnapshotDirectoryCleaningService.java | 6 +-
.../reconfig/TestDatanodeReconfiguration.java | 2 +
.../shell/TestOzoneContainerUpgradeShell.java | 6 +-
.../ozone/shell/TestOzoneDebugReplicasVerify.java | 2 +-
.../hadoop/ozone/shell/TestOzoneDebugShell.java | 6 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 185 +-
.../ozone/shell/TestOzoneShellHAWithFSO.java | 43 -
.../shell/TestOzoneShellHAWithFollowerRead.java | 42 +-
.../shell/TestReplicationConfigPreference.java | 12 +-
.../java/org/apache/ozone/test/NonHATests.java | 9 +
.../src/main/proto/OmClientProtocol.proto | 193 +-
.../interface-client/src/main/resources/proto.lock | 18 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 30 +-
.../hadoop/ozone/om/lock/IOzoneManagerLock.java | 46 +-
.../org/apache/hadoop/ozone/FixedHostMapping.java | 153 +
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 50 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 92 +
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 17 +-
.../hadoop/ozone/UniformDatanodesFactory.java | 6 +
.../RangerClientMultiTenantAccessController.java | 98 +-
...lientMultiTenantAccessControllerMockClient.java | 153 +
hadoop-ozone/ozone-manager/pom.xml | 11 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 8 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 64 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 201 +-
.../org/apache/hadoop/ozone/om/ListIterator.java | 3 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 8 +-
.../om/OMDBCheckpointServletInodeBasedXfer.java | 8 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 56 +
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 4 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 7 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 104 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 37 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 32 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 4 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 342 +-
.../hadoop/ozone/om/SnapshotChainManager.java | 2 +-
.../hadoop/ozone/om/SstFilteringService.java | 3 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 1 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 47 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 2 +-
.../hadoop/ozone/om/helpers/OMAuditLogger.java | 1 +
.../ozone/om/lock/DAGResourceLockTracker.java | 5 +
.../ozone/om/lock/LeveledResourceLockTracker.java | 6 +
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 1 +
.../hadoop/ozone/om/lock/OmReadOnlyLock.java | 58 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 512 +-
.../hadoop/ozone/om/lock/ResourceLockTracker.java | 2 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 12 +
.../ozone/om/request/OMClientRequestUtils.java | 2 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 18 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 4 +-
.../ozone/om/request/file/OMFileRequest.java | 2 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 2 +
.../om/request/key/OMKeyRenameRequestWithFSO.java | 8 +
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 62 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 109 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 6 +-
.../OMLifecycleConfigurationDeleteRequest.java | 209 +
.../OMLifecycleConfigurationSetRequest.java | 238 +
.../lifecycle/OMLifecycleSaveScanStateRequest.java | 74 +
.../OMLifecycleSetServiceStatusRequest.java | 125 +
.../ozone/om/request/lifecycle/package-info.java | 37 +-
.../S3InitiateMultipartUploadRequest.java | 39 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 3 +
.../S3MultipartUploadCommitPartRequest.java | 18 +-
.../S3MultipartUploadCompleteRequest.java | 7 +
.../request/snapshot/OMSnapshotDeleteRequest.java | 8 +
.../om/request/util/OMMultipartUploadUtils.java | 13 +-
.../om/request/volume/OMQuotaRepairRequest.java | 11 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 11 +-
.../om/response/key/OMKeysDeleteResponse.java | 17 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 15 +-
.../OMLifecycleConfigurationDeleteResponse.java | 64 +
.../OMLifecycleConfigurationSetResponse.java | 67 +
.../OMLifecycleSaveScanStateResponse.java | 51 +
.../OMLifecycleSetServiceStatusResponse.java | 46 +
.../ozone/om/response/lifecycle/package-info.java | 37 +-
.../snapshot/OMSnapshotMoveTableKeysResponse.java | 7 +-
.../hadoop/ozone/om/service/CompactDBUtil.java | 22 +-
.../hadoop/ozone/om/service/CompactionService.java | 12 +-
.../ozone/om/service/DirectoryDeletingService.java | 5 +-
.../ozone/om/service/KeyDeletingService.java | 15 +-
.../ozone/om/service/KeyLifecycleService.java | 2172 +++++
.../om/service/KeyLifecycleServiceMetrics.java | 196 +
.../ozone/om/service/OMRangerBGSyncService.java | 4 +-
.../om/snapshot/FSODirectoryPathResolver.java | 3 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 168 +-
.../om/snapshot/defrag/SnapshotDefragService.java | 147 +-
.../diff/delta/CompositeDeltaDiffComputer.java | 3 +
.../om/snapshot/filter/ReclaimableFilter.java | 15 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 40 +
.../apache/hadoop/ozone/om/FaultInjectorImpl.java | 98 +
.../hadoop/ozone/om/TestBucketManagerImpl.java | 212 +
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 61 +-
...DBCheckpointServletInodeBasedXferNonLeader.java | 86 +-
.../hadoop/ozone/om/TestOMMetadataReader.java | 2 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 231 +-
.../hadoop/ozone/om/lock/TestKeyPathLock.java | 24 +-
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 150 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 279 +
.../om/request/key/TestOMKeyCommitRequest.java | 4 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 158 +-
.../key/TestOMKeysDeleteRequestWithFSO.java | 77 +-
.../TestOMLifecycleConfigurationDeleteRequest.java | 296 +
.../TestOMLifecycleConfigurationRequest.java | 153 +
.../TestOMLifecycleConfigurationSetRequest.java | 386 +
.../TestOMLifecycleSaveScanStateRequest.java | 117 +
.../s3/multipart/S3MultipartRequestTests.java | 64 +-
.../TestS3InitiateMultipartUploadRequest.java | 108 +
.../TestS3MultipartUploadAbortRequest.java | 88 +
.../TestS3MultipartUploadCommitPartRequest.java | 125 +-
.../TestS3MultipartUploadCompleteRequest.java | 79 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 6 +-
.../snapshot/TestOMSnapshotDeleteRequest.java | 49 +
.../request/volume/TestOMQuotaRepairRequest.java | 95 +
.../ozone/om/response/TestCleanupTableInfo.java | 2 +
.../om/response/key/TestOMKeyCommitResponse.java | 2 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 2 +-
.../om/response/key/TestOMKeysDeleteResponse.java | 2 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 2 +-
...TestOMLifecycleConfigurationDeleteResponse.java | 117 +
.../TestOMLifecycleConfigurationSetResponse.java | 124 +
.../TestOMLifecycleSaveScanStateResponse.java | 61 +
.../TestS3MultipartUploadCommitPartResponse.java | 2 +-
.../TestOMSnapshotMoveTableKeysResponse.java | 6 +-
.../hadoop/ozone/om/service/TestCompactDBUtil.java | 92 +
.../ozone/om/service/TestCompactionService.java | 33 +
.../ozone/om/service/TestKeyDeletingService.java | 247 +-
.../ozone/om/service/TestKeyLifecycleService.java | 3673 ++++++++
.../om/snapshot/TestFSODirectoryPathResolver.java | 2 +-
.../ozone/om/snapshot/TestSnapshotCache.java | 16 +
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 162 +-
.../om/snapshot/TestSnapshotDiffManagerMXBean.java | 4 +-
.../snapshot/defrag/TestSnapshotDefragService.java | 319 +-
.../snapshot/filter/TestReclaimableKeyFilter.java | 71 +
.../om/upgrade/OMVersionManagerTestUtils.java | 15 +
.../ozone/om/upgrade/TestOMLayoutFeature.java | 6 +-
.../protocolPB/TestOzoneManagerRequestHandler.java | 64 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 9 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 39 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 12 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 29 +-
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 8 +-
.../ozone/TestBasicOzoneClientAdapterHeadOp.java | 136 +
.../hadoop/fs/ozone/TestBasicOzoneFileSystems.java | 49 +
.../hadoop/fs/ozone/TestOzoneFileSystemHeadOp.java | 149 +
.../recon/schema/ContainerSchemaDefinition.java | 12 +-
hadoop-ozone/recon/pom.xml | 4 -
.../hadoop/ozone/recon/ReconRestServletModule.java | 41 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 10 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 57 +-
.../ozone/recon/api/OMDBInsightEndpoint.java | 3 +-
.../ozone/recon/api/ReconGlobalMetricsService.java | 4 +-
.../ozone/recon/api/TriggerDBSyncEndpoint.java | 12 +
.../ozone/recon/api/handlers/EntityHandler.java | 122 +-
.../ozone/recon/api/handlers/FSOBucketHandler.java | 3 +-
.../recon/api/handlers/LegacyBucketHandler.java | 5 +-
.../ozone/recon/api/handlers/OBSBucketHandler.java | 8 +-
.../recon/api/types/OMDBReprocessResponse.java | 59 +-
.../ozone/recon/chatbot/ChatbotConfigKeys.java | 35 +
.../recon/chatbot/agent/ChatHistoryBuilder.java | 175 +
.../ozone/recon/chatbot/agent/ChatbotAgent.java | 90 +-
.../ozone/recon/chatbot/api/ChatbotEndpoint.java | 63 +-
.../recon/chatbot/llm/LangChain4jDispatcher.java | 27 +
.../ozone/recon/metrics/NSSummaryMetrics.java | 91 +
.../recon/recovery/ReconOmMetadataManagerImpl.java | 9 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 6 +-
.../scm/ReconStorageContainerManagerFacade.java | 3 +-
.../recon/spi/OzoneManagerServiceProvider.java | 7 +
.../recon/spi/ReconNamespaceSummaryManager.java | 6 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 237 +-
.../impl/ReconContainerMetadataManagerImpl.java | 21 +-
.../ozone/recon/spi/impl/ReconDBProvider.java | 16 -
.../spi/impl/ReconFileMetadataManagerImpl.java | 5 +-
.../spi/impl/ReconNamespaceSummaryManagerImpl.java | 24 +-
.../recon/spi/impl/ReconRDBSnapshotProvider.java | 245 +
.../impl/StorageContainerServiceProviderImpl.java | 3 +-
.../recon/tasks/ContainerKeyMapperHelper.java | 2 +-
.../recon/tasks/DeletedKeysInsightHandler.java | 2 +-
.../recon/tasks/MultipartInfoInsightHandler.java | 219 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 3 +-
.../ozone/recon/tasks/OpenKeysInsightHandler.java | 2 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 158 +-
.../tasks/ReconTaskReInitializationEvent.java | 3 +-
.../tasks/util/ParallelTableIteratorOperation.java | 4 +-
.../ozone/recon/upgrade/ReconUpgradeAction.java | 12 +-
.../recon-tool-selection-prompt-preamble.txt | 7 +
.../webapps/recon/ozone-recon-web/package.json | 6 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 1428 +--
.../src/__tests__/Overview.test.tsx | 80 +-
.../src/__tests__/assistant/Assistant.test.tsx | 30 +-
.../src/__tests__/capacity/Capacity.test.tsx | 330 +-
.../src/__tests__/datanodes/Datanodes.test.tsx | 27 +-
.../__tests__/datanodes/DatanodesTable.test.tsx | 32 +-
.../src/__tests__/locators/locators.ts | 6 +-
.../mocks/assistantMocks/assistantServer.ts | 168 +-
.../mocks/capacityMocks/capacityServer.ts | 34 +-
.../mocks/datanodeMocks/datanodeResponseMocks.ts | 17 +-
.../mocks/datanodeMocks/datanodeServer.ts | 44 +-
.../mocks/overviewMocks/overviewResponseMocks.ts | 43 +
.../mocks/overviewMocks/overviewServer.ts | 58 +-
.../mocks/pipelineMocks/pipelinesServer.ts | 9 +-
.../src/__tests__/pipelines/Pipelines.test.tsx | 30 +-
.../src/v2/components/cards/overviewHealthCard.tsx | 5 +-
.../src/v2/components/storageBar/storageBar.tsx | 41 +-
.../ozone-recon-web/src/v2/hooks/useChat.hook.tsx | 28 +-
.../src/v2/pages/capacity/capacity.tsx | 92 +-
.../pages/capacity/components/CapacityDetail.tsx | 4 +-
.../src/v2/pages/datanodes/datanodes.tsx | 6 +-
.../src/v2/pages/overview/overview.tsx | 3 +-
.../src/v2/pages/pipelines/pipelines.tsx | 2 +-
.../ozone-recon-web/src/v2/types/chatbot.types.ts | 8 +
.../ozone/recon/OMMetadataManagerTestUtils.java | 36 +
.../apache/hadoop/ozone/recon/TestReconUtils.java | 40 +
.../recon/api/AbstractTreeNSSummaryScenario.java | 806 ++
.../ozone/recon/api/FlatNSSummaryScenario.java | 655 ++
.../ozone/recon/api/FsoNSSummaryScenario.java | 153 +
.../ozone/recon/api/LegacyNSSummaryScenario.java | 130 +
.../ozone/recon/api/NSSummaryEndpointTestBase.java | 236 +
.../ozone/recon/api/NSSummaryTestScenario.java | 163 +
.../hadoop/ozone/recon/api/NSSummaryTests.java | 195 -
.../ozone/recon/api/TestNSSummaryEndpoint.java | 389 +
.../recon/api/TestNSSummaryEndpointWithFSO.java | 1577 ----
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 1491 ----
.../api/TestNSSummaryEndpointWithOBSAndLegacy.java | 1452 ---
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 1 +
.../api/handlers/TestEntityHandlerCycleGuard.java | 196 +
.../chatbot/agent/TestChatHistoryBuilder.java | 289 +
.../agent/TestChatbotAgentExecutionPolicy.java | 62 +
.../recon/chatbot/api/TestChatbotEndpoint.java | 14 +-
.../chatbot/llm/TestLangChain4jDispatcher.java | 57 +
.../ozone/recon/chatbot/llm/TestLlmRouting.java | 8 +
.../ozone/recon/metrics/TestNSSummaryMetrics.java | 60 +
.../impl/TestOzoneManagerServiceProviderImpl.java | 321 +-
.../TestReconContainerMetadataManagerImpl.java | 22 +
.../spi/impl/TestReconRDBSnapshotProvider.java | 223 +
.../ozone/recon/tasks/TestFileSizeCountTask.java | 23 +-
.../ozone/recon/tasks/TestOmTableInsightTask.java | 129 +
.../recon/tasks/TestReconTaskControllerImpl.java | 160 +-
hadoop-ozone/s3gateway/pom.xml | 5 -
.../org/apache/hadoop/ozone/audit/S3GAction.java | 7 +-
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 37 +-
.../ozone/s3/commontypes/RequestParameters.java | 14 +
.../ozone/s3/endpoint/BucketCrudHandler.java | 3 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 23 +-
.../ozone/s3/endpoint/BucketLifecycleHandler.java | 170 +
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 106 +-
.../s3/endpoint/GetObjectAttributesResponse.java | 206 +
.../ozone/s3/endpoint/ObjectAttributesHandler.java | 173 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 138 +-
.../ozone/s3/endpoint/ObjectGetTorrentHandler.java | 48 +
...utBucketLifecycleConfigurationUnmarshaller.java | 80 +
.../s3/endpoint/S3LifecycleConfiguration.java | 558 ++
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 7 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 22 +
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 14 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 19 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 29 +
.../hadoop/ozone/client/OzoneBucketStub.java | 68 +
.../hadoop/ozone/client/OzoneVolumeStub.java | 2 +
.../ozone/s3/TestVirtualHostStyleFilter.java | 204 +-
.../ozone/s3/endpoint/EndpointTestUtils.java | 14 +
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 37 +
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 60 +-
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 26 +
.../s3/endpoint/TestMultipartUploadComplete.java | 22 +
.../ozone/s3/endpoint/TestObjectAttributesGet.java | 176 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 67 +-
...estBucketPut.java => TestObjectGetTorrent.java} | 40 +-
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 99 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 87 +
.../TestS3LifecycleConfigurationDelete.java | 96 +
.../endpoint/TestS3LifecycleConfigurationGet.java | 205 +
.../endpoint/TestS3LifecycleConfigurationPut.java | 665 ++
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 20 +
hadoop-ozone/tools/pom.xml | 15 +
.../hadoop/ozone/local/LocalOzoneCluster.java | 228 +-
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 64 +-
.../hadoop/ozone/local/TestLocalOzoneCluster.java | 109 +
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 120 +-
hadoop-ozone/vapor/pom.xml | 11 +
pom.xml | 217 +-
763 files changed, 61265 insertions(+), 12738 deletions(-)
diff --cc hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java
index 4b9dd57004c,221cae5f62e..08c9f45f575
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java
@@@ -18,6 -18,7 +18,7 @@@
package org.apache.hadoop.hdds.scm;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.apache.hadoop.hdds.DatanodeVersion.SHORT_CIRCUIT_READS;
++import static org.apache.hadoop.hdds.HDDSVersion.SHORT_CIRCUIT_READS;
import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
import static org.apache.hadoop.hdds.conf.ConfigTag.PERFORMANCE;
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.NO_REPLICA_FOUND;
@@@ -161,13 -194,35 +194,34 @@@ public class XceiverClientManager exten
}
}
- private String getPipelineCacheKey(Pipeline pipeline,
- boolean topologyAware) {
+ private String getPipelineCacheKey(Pipeline pipeline, boolean topologyAware, boolean allowShortCircuit) {
StringBuilder key = new StringBuilder()
- .append(pipeline.getId().getId())
- .append(pipeline.getType());
+ .append(pipeline.getId().getId()).append('-').append(pipeline.getType());
boolean isEC = pipeline.getType() == HddsProtos.ReplicationType.EC;
- if (topologyAware || isEC) {
+ DatanodeDetails localDN = null;
+ boolean shortCircuitPipeline = false;
+
+ if ((!isEC) && allowShortCircuit && isShortCircuitEnabled()) {
+ int port = 0;
+ InetSocketAddress localAddr = null;
+ for (DatanodeDetails dn : pipeline.getNodes()) {
+ // read port from the data node, on failure use default configured port.
+ port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
+ InetSocketAddress addr = NetUtils.createSocketAddr(dn.getIpAddress(), port);
- if (OzoneNetUtils.isAddressLocal(addr) &&
- dn.getCurrentVersion() >= SHORT_CIRCUIT_READS.toProtoValue()) {
++ if (OzoneNetUtils.isAddressLocal(addr) && SHORT_CIRCUIT_READS.isSupportedBy(dn.getCurrentVersion())) {
+ localAddr = addr;
+ localDN = dn;
+ break;
+ }
+ }
+ if (localAddr != null) {
+ // Find a local DN and short circuit read is enabled
+ key.append('@').append(localAddr.getHostName()).append(':').append(port);
+ shortCircuitPipeline = true;
+ }
+ }
+
+ if (localDN == null && (topologyAware || isEC)) {
try {
DatanodeDetails closestNode = pipeline.getClosestNode();
// Pipeline cache key uses host:port suffix to handle
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java
index b9de634c48e,560a1edf123..1a3e2223436
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java
@@@ -38,19 -35,18 +38,20 @@@ public enum HDDSVersion implements Comp
"a PutBlock request"),
STREAM_BLOCK_SUPPORT(3,
"This version has support for reading a block by streaming chunks."),
-
+ SHORT_CIRCUIT_READS(4, "Version with short-circuit read support."),
- FUTURE_VERSION(-1, "Used internally in the client when the server side is "
- + " newer and an unknown server version has arrived to the client.");
+ ZDU(100, "Version that supports zero downtime upgrade"),
- public static final DatanodeVersion CURRENT = latest();
- public static final int CURRENT_VERSION = CURRENT.version;
+ UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
+ " component, which may indicate it came from a component in a newer version");
- private static final Map<Integer, DatanodeVersion> BY_PROTO_VALUE =
+ ////////////////////////////// //////////////////////////////
+
+ private static final SortedMap<Integer, HDDSVersion> BY_VALUE =
Arrays.stream(values())
- .collect(toMap(DatanodeVersion::toProtoValue, identity()));
+ .collect(toMap(HDDSVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new));
+
+ public static final HDDSVersion SOFTWARE_VERSION = BY_VALUE.get(BY_VALUE.lastKey());
private final int version;
private final String description;
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
index 66856efcb4e,2ea4fa64213..8925d325942
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
@@@ -578,10 -585,11 +591,10 @@@ public class DatanodeDetails extends No
builder.setPersistedOpStateExpiry(persistedOpStateExpiryEpochSec);
final boolean handlesUnknownPorts =
- ClientVersion.fromProtoValue(clientVersion)
- .compareTo(VERSION_HANDLES_UNKNOWN_DN_PORTS) >= 0;
+ VERSION_HANDLES_UNKNOWN_DN_PORTS.isSupportedBy(clientVersion);
final int requestedPortCount = filterPorts.size();
final boolean maySkip = requestedPortCount > 0;
- for (Port port : ports) {
+ for (Port port : ports.values()) {
if (maySkip && !filterPorts.contains(port.getName())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip adding {} port {} to proto message",
diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml
index 58379a2a6dc,3e51a6d9d04..54e1225f068
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@@ -5163,10 -5203,77 +5203,82 @@@
<tag>OZONE, RATIS, OM</tag>
<description>The maximum number of events that can be pending in OM Ratis.</description>
</property>
-
+ <property>
+ <name>ozone.om.upgrade.finalization.check.interval</name>
+ <value>1m</value>
+ <tag>OM</tag>
+ <description>If OM is unfinalized, how frequently it should poll SCM to trigger finalization</description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.enabled</name>
+ <value>false</value>
+ <tag>OZONE</tag>
+ <description>It specifies whether to enable lifecycle management service.</description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.move.to.trash.enabled</name>
+ <value>true</value>
+ <tag>OZONE</tag>
+ <description>When enabled KeyLifecycleService will move expired keys/dirs to trash if trash is available.
+ When disabled, it will delete them directly.
+ </description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.delete.batch-size</name>
+ <value>1000</value>
+ <tag>OZONE</tag>
+ <description>Max numbers of objects allowed for deletion in a batch for a lifecycle evaluating task.</description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.interval</name>
+ <value>24h</value>
+ <tag>OZONE</tag>
+ <description>Interval at which key lifecycle management service will run.</description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.timeout</name>
+ <value>2h</value>
+ <tag>OZONE</tag>
+ <description>Timeout for task of key lifecycle management service.</description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.workers</name>
+ <value>5</value>
+ <tag>OZONE</tag>
+ <description>Number of workers executed of key lifecycle management service. This
+ configuration should be set to greater than 0.</description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.delete.cached.directory.max-count</name>
+ <value>1000000</value>
+ <tag>OZONE</tag>
+ <description>Max numbers of directory objects held in memory stack for recursive FSO bucket evaluating for
+ a lifecycle evaluating task. Once the cached directory objects exceeds this limit, the evaluation of the involved
+ directories will abort.
+ </description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.mpu.abort.limit.per.task</name>
+ <value>1000</value>
+ <tag>OZONE</tag>
+ <description>Maximum number of multipart upload parts (rounded up to complete uploads)
+ to abort in a single task when processing AbortIncompleteMultipartUpload
+ lifecycle rules.
+ </description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.state.save.interval.ms</name>
+ <value>300000</value>
+ <tag>OZONE</tag>
+ <description>The interval of bucket scan task saves its pointer to DB. Default is 5 mins.</description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.state.save.keys.processed</name>
+ <value>100000</value>
+ <tag>OZONE</tag>
+ <description>Bucket scan task will save its pointer to DB by default every 100000 keys are scanned. Bucket scan
+ pointer save will happen when either ozone.lifecycle.service.state.save.interval.ms or
+ ozone.lifecycle.service.state.save.keys.processed is satisfied.
+ </description>
+ </property>
</configuration>
diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/TestDatanodeDetails.java
index 2df255aae1d,8994044e4b6..2a6416b0185
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/TestDatanodeDetails.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/TestDatanodeDetails.java
@@@ -73,19 -75,54 +75,54 @@@ public class TestDatanodeDetails
DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
Set<Port.Name> requiredPorts = Stream.of(Port.Name.STANDALONE, Port.Name.RATIS)
.collect(Collectors.toSet());
- HddsProtos.DatanodeDetailsProto.Builder protoBuilder =
- dn.toProtoBuilder(DEFAULT_VERSION.toProtoValue(), requiredPorts);
+ HddsProtos.DatanodeDetailsProto.Builder protoBuilder = dn.toProtoBuilder(ClientVersion.CURRENT, requiredPorts);
protoBuilder.clearCurrentVersion();
DatanodeDetails dn2 = DatanodeDetails.newBuilder(protoBuilder.build()).build();
- assertEquals(DatanodeVersion.SEPARATE_RATIS_PORTS_AVAILABLE.toProtoValue(), dn2.getCurrentVersion());
+ assertEquals(HDDSVersion.DEFAULT_VERSION, dn2.getCurrentVersion());
- // test that if the current version is set, it is used
- protoBuilder =
- dn.toProtoBuilder(DEFAULT_VERSION.toProtoValue(), requiredPorts);
- DatanodeDetails dn3 = DatanodeDetails.newBuilder(protoBuilder.build()).build();
- assertEquals(DatanodeVersion.CURRENT.toProtoValue(), dn3.getCurrentVersion());
+ // When the proto field is present, it round-trips correctly.
+ protoBuilder = dn.toProtoBuilder(ClientVersion.CURRENT, requiredPorts);
+ DatanodeDetails dn3 = DatanodeDetails.newBuilder(
+ protoBuilder.setCurrentVersion(HDDSVersion.SOFTWARE_VERSION.serialize()).build())
+ .build();
+ assertEquals(HDDSVersion.SOFTWARE_VERSION, dn3.getCurrentVersion());
}
+ @Test
+ void portsChangedComparesNameAndValue() {
+ DatanodeID id = DatanodeID.randomID();
+ DatanodeDetails base = DatanodeDetails.newBuilder()
+ .setID(id)
+ .addPort(DatanodeDetails.newStandalonePort(9858))
+ .addPort(DatanodeDetails.newRatisPort(9859))
+ .build();
+
+ // Identical name/value set: no change.
+ DatanodeDetails same = DatanodeDetails.newBuilder()
+ .setID(id)
+ .addPort(DatanodeDetails.newStandalonePort(9858))
+ .addPort(DatanodeDetails.newRatisPort(9859))
+ .build();
+ assertFalse(base.portsChanged(same));
+
+ // Same names, one different value: detected (Port.equals ignores value).
+ DatanodeDetails changedValue = DatanodeDetails.newBuilder()
+ .setID(id)
+ .addPort(DatanodeDetails.newStandalonePort(9858))
+ .addPort(DatanodeDetails.newRatisPort(1234))
+ .build();
+ assertTrue(base.portsChanged(changedValue));
+
+ // Extra port: detected (key set differs).
+ DatanodeDetails extraPort = DatanodeDetails.newBuilder()
+ .setID(id)
+ .addPort(DatanodeDetails.newStandalonePort(9858))
+ .addPort(DatanodeDetails.newRatisPort(9859))
+ .addPort(DatanodeDetails.newPort(Name.RATIS_DATASTREAM, 9860))
+ .build();
+ assertTrue(base.portsChanged(extraPort));
+ }
+
public static void assertPorts(HddsProtos.DatanodeDetailsProto dn,
Set<Port.Name> expectedPorts) throws IllegalArgumentException {
assertEquals(expectedPorts.size(), dn.getPortsCount());
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index e7bed34a6f7,6c03d140d84..100070b19d5
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@@ -19,9 -19,13 +19,13 @@@ package org.apache.hadoop.ozone.contain
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT;
+ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD;
+ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT;
+ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_DEFAULT;
+ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY;
-import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto;
+import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto;
import com.google.common.base.Preconditions;
import com.google.protobuf.Descriptors;
@@@ -43,6 -47,8 +47,7 @@@ import org.apache.hadoop.hdds.protocol.
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+ import org.apache.hadoop.hdds.utils.ConnectionFailureUtils;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.ozone.container.common.helpers.DeletedContainerBlocksSummary;
import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
@@@ -76,7 -81,9 +81,9 @@@ public class HeartbeatEndpointTas
private StateContext context;
private int maxContainerActionsPerHB;
private int maxPipelineActionsPerHB;
- private HDDSLayoutVersionManager layoutVersionManager;
+ private final DatanodeVersionManager versionManager;
+ private final boolean resolveOnFailureEnabled;
+ private final int refreshThreshold;
/**
* Constructs a SCM heart beat.
@@@ -93,10 -102,36 +100,14 @@@
HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT);
this.maxPipelineActionsPerHB = conf.getInt(HDDS_PIPELINE_ACTION_MAX_LIMIT,
HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT);
- if (versionManager != null) {
- this.layoutVersionManager = versionManager;
- } else {
- this.layoutVersionManager = context.getParent().getLayoutVersionManager();
- }
+ this.datanodeDetails = context.getParent().getDatanodeDetails();
+ this.versionManager = context.getParent().getVersionManager();
+ this.resolveOnFailureEnabled = conf.getBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY,
+ OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_DEFAULT);
+ this.refreshThreshold = Math.max(1, conf.getInt(HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD,
+ HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD_DEFAULT));
}
- /**
- * Get the container Node ID proto.
- *
- * @return ContainerNodeIDProto
- */
- public DatanodeDetailsProto getDatanodeDetailsProto() {
- return datanodeDetailsProto;
- }
-
- /**
- * Set container node ID proto.
- *
- * @param datanodeDetailsProto - the node id.
- */
- public void setDatanodeDetailsProto(DatanodeDetailsProto
- datanodeDetailsProto) {
- this.datanodeDetailsProto = datanodeDetailsProto;
- }
-
/**
* Computes a result, or throws an exception if unable to do so.
*
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTaskDnsRefresh.java
index 00000000000,6d9a4406c5d..e8c3f391108
mode 000000,100644..100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTaskDnsRefresh.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTaskDnsRefresh.java
@@@ -1,0 -1,133 +1,136 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.ozone.container.common.states.endpoint;
+
+ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD;
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
+ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY;
+ import static org.mockito.Mockito.any;
+ import static org.mockito.Mockito.eq;
+ import static org.mockito.Mockito.mock;
+ import static org.mockito.Mockito.never;
+ import static org.mockito.Mockito.times;
+ import static org.mockito.Mockito.verify;
+ import static org.mockito.Mockito.when;
+
+ import java.io.IOException;
+ import java.net.ConnectException;
+ import java.util.UUID;
++import org.apache.hadoop.hdds.HDDSVersion;
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+ import org.apache.hadoop.hdds.scm.net.HostAndPort;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+ import org.apache.hadoop.hdfs.util.EnumCounters;
+ import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+ import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates;
+ import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
+ import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
+ import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
++import org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager;
+ import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolClientSideTranslatorPB;
+ import org.junit.jupiter.api.Test;
+
+ /**
+ * Verifies a connection-class heartbeat failure past the threshold triggers a DNS re-resolution of
+ * the SCM peer (HDDS-15533); flag-off, application errors, and below-threshold do not.
+ */
+ public class TestHeartbeatEndpointTaskDnsRefresh {
+
+ private static final HostAndPort SCM = new HostAndPort("test-scm-1", 9861);
+
+ @Test
+ public void connectionFailureAtThresholdTriggersRefresh() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY, true);
+ conf.setInt(HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD, 2);
+ SCMConnectionManager cm = runHeartbeat(conf, 3, new ConnectException("refused"));
+ verify(cm, times(1)).refreshSCMServer(eq(SCM), any());
+ }
+
+ @Test
+ public void flagOffSuppressesRefresh() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY, false);
+ SCMConnectionManager cm = runHeartbeat(conf, 5, new ConnectException("refused"));
+ verify(cm, never()).refreshSCMServer(any(), any());
+ }
+
+ @Test
+ public void applicationErrorDoesNotTriggerRefresh() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY, true);
+ SCMConnectionManager cm = runHeartbeat(conf, 5, new IOException("application-level"));
+ verify(cm, never()).refreshSCMServer(any(), any());
+ }
+
+ @Test
+ public void belowThresholdDoesNotTriggerRefresh() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_CLIENT_FAILOVER_RESOLVE_NEEDED_KEY, true);
+ conf.setInt(HDDS_HEARTBEAT_ADDRESS_REFRESH_MISSED_COUNT_THRESHOLD, 5);
+ SCMConnectionManager cm = runHeartbeat(conf, 1, new ConnectException("refused"));
+ verify(cm, never()).refreshSCMServer(any(), any());
+ }
+
+ /**
+ * Drives one heartbeat that fails with {@code failure}, with the endpoint reporting
+ * {@code missedCount} missed heartbeats, and returns the mocked connection manager to verify.
+ */
+ private SCMConnectionManager runHeartbeat(OzoneConfiguration conf, long missedCount,
+ IOException failure) throws Exception {
+ StorageContainerDatanodeProtocolClientSideTranslatorPB proxy =
+ mock(StorageContainerDatanodeProtocolClientSideTranslatorPB.class);
+ when(proxy.sendHeartbeat(any())).thenThrow(failure);
+
+ EndpointStateMachine endpoint = mock(EndpointStateMachine.class);
+ when(endpoint.getEndPoint()).thenReturn(proxy);
+ when(endpoint.getAddress()).thenReturn(SCM);
+ when(endpoint.getMissedCount()).thenReturn(missedCount);
+ when(endpoint.isPassive()).thenReturn(false);
+
+ SCMConnectionManager connectionManager = mock(SCMConnectionManager.class);
+ DatanodeStateMachine dsm = mock(DatanodeStateMachine.class);
+ when(dsm.getConnectionManager()).thenReturn(connectionManager);
+ when(dsm.getQueuedCommandCount())
+ .thenReturn(new EnumCounters<>(SCMCommandProto.Type.class));
- StateContext context = new StateContext(conf, DatanodeStates.RUNNING, dsm, "");
-
- HDDSLayoutVersionManager lvm = mock(HDDSLayoutVersionManager.class);
- when(lvm.getSoftwareLayoutVersion()).thenReturn(maxLayoutVersion());
- when(lvm.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
+
+ DatanodeDetails dn = DatanodeDetails.newBuilder()
+ .setUuid(UUID.randomUUID())
+ .setHostName("localhost")
+ .setIpAddress("127.0.0.1")
++ .setCurrentVersion(HDDSVersion.SOFTWARE_VERSION)
+ .build();
++ when(dsm.getDatanodeDetails()).thenReturn(dn);
++
++ DatanodeVersionManager versionManager = mock(DatanodeVersionManager.class);
++ when(versionManager.getApparentVersion()).thenReturn(HDDSVersion.SOFTWARE_VERSION);
++ when(versionManager.getSoftwareVersion()).thenReturn(HDDSVersion.SOFTWARE_VERSION);
++ when(versionManager.getVersionForClient()).thenReturn(HDDSVersion.SOFTWARE_VERSION);
++ when(dsm.getVersionManager()).thenReturn(versionManager);
++
++ StateContext context = new StateContext(conf, DatanodeStates.RUNNING, dsm, "");
+
+ HeartbeatEndpointTask.newBuilder()
+ .setConfig(conf)
- .setDatanodeDetails(dn)
+ .setContext(context)
- .setLayoutVersionManager(lvm)
+ .setEndpointStateMachine(endpoint)
+ .build()
+ .call();
+ return connectionManager;
+ }
+ }
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
index 5f8e9a754cd,21efce17dee..2489c02d1ab
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@@ -26,10 -26,17 +26,17 @@@ import static org.apache.hadoop.hdds.pr
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicationCommandPriority.LOW;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicationCommandPriority.NORMAL;
++import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.getReplicateContainerCommand;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
import static org.apache.hadoop.ozone.container.replication.AbstractReplicationTask.Status.DONE;
+ import static org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig.PER_VOLUME_ENABLED_KEY;
+ import static org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig.PER_VOLUME_STREAMS_LIMIT_KEY;
-import static org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand.toTarget;
+ import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertFalse;
+ import static org.junit.jupiter.api.Assertions.assertNotNull;
+ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyLong;
@@@ -800,7 -819,8 +819,7 @@@ public class TestReplicationSupervisor
}
private ReplicateContainerCommand createCommand(long containerId) {
- ReplicateContainerCommand cmd = ContainerTestUtils.getReplicateContainerCommand(containerId, datanode);
- ReplicateContainerCommand cmd =
- ReplicateContainerCommand.toTarget(containerId, datanode);
++ ReplicateContainerCommand cmd = getReplicateContainerCommand(containerId, datanode);
cmd.setTerm(CURRENT_TERM);
return cmd;
}
@@@ -1048,7 -1068,455 +1067,455 @@@
List<DatanodeDetails> datanodes, ReplicationSupervisor rs) {
for (int i = 0; i < 10; i++) {
DatanodeDetails target = datanodes.get(i % datanodes.size());
- rs.addTask(new ReplicationTask(ContainerTestUtils.getReplicateContainerCommand(i, target), noopReplicator));
- rs.addTask(new ReplicationTask(toTarget(i, target), noopReplicator));
++ rs.addTask(new ReplicationTask(getReplicateContainerCommand(i, target), noopReplicator));
}
}
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void perVolumeDisabledUsesGlobalPool(ContainerLayoutVersion layout) {
+ this.layoutVersion = layout;
+ ReplicationServer.ReplicationConfig repConf =
+ new ReplicationServer.ReplicationConfig();
+ repConf.setPerVolumeEnabled(false);
+
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .executor(newDirectExecutorService())
+ .clock(clock)
+ .build();
+
+ try {
+ assertNull(supervisor.getVolumeReplicationThreadPools());
+ replicatorRef.set(doneReplicator);
+ supervisor.addTask(createTask(1L));
+ assertEquals(1, supervisor.getReplicationSuccessCount());
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void perVolumeInitLogging(ContainerLayoutVersion layout,
+ @TempDir File perVolumeTempDir) throws Exception {
+ this.layoutVersion = layout;
+ OzoneConfiguration conf = perVolumeConf(perVolumeTempDir, 1);
+ MutableVolumeSet volumeSet = newVolumeSet(conf);
+ ReplicationServer.ReplicationConfig repConf =
+ conf.getObject(ReplicationServer.ReplicationConfig.class);
+
+ LogCapturer supervisorLogs =
+ LogCapturer.captureLogs(ReplicationSupervisor.class);
+ LogCapturer poolLogs =
+ LogCapturer.captureLogs(VolumeReplicationThreadPools.class);
+
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .containerSet(set)
+ .volumeSet(volumeSet)
+ .executor(newDirectExecutorService())
+ .clock(clock)
+ .build();
+
+ try {
+ assertNotNull(supervisor.getVolumeReplicationThreadPools());
+ assertThat(supervisorLogs.getOutput())
+ .contains("Per-volume container replication thread pools enabled");
+ assertThat(poolLogs.getOutput())
+ .contains("Initialized 2 per-volume replication thread pools");
+ for (StorageVolume volume : volumeSet.getVolumesList()) {
+ assertThat(poolLogs.getOutput())
+ .contains(volume.getStorageDir().getPath());
+ }
+ } finally {
+ supervisorLogs.stopCapturing();
+ poolLogs.stopCapturing();
+ supervisor.stop();
+ }
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void perVolumePoolSizeRespected(ContainerLayoutVersion layout,
+ @TempDir File perVolumeTempDir) throws Exception {
+ this.layoutVersion = layout;
+ OzoneConfiguration conf = perVolumeConf(perVolumeTempDir, 3);
+ MutableVolumeSet volumeSet = newVolumeSet(conf);
+ ReplicationServer.ReplicationConfig repConf =
+ conf.getObject(ReplicationServer.ReplicationConfig.class);
+
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .containerSet(set)
+ .volumeSet(volumeSet)
+ .executor(newDirectExecutorService())
+ .clock(clock)
+ .build();
+
+ try {
+ VolumeReplicationThreadPools pools =
+ supervisor.getVolumeReplicationThreadPools();
+ assertNotNull(pools);
+ for (StorageVolume volume : volumeSet.getVolumesList()) {
+ assertEquals(3, pools.getPoolSize(volume.getStorageDir().getPath()));
+ }
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void perVolumePoolResize(ContainerLayoutVersion layout,
+ @TempDir File perVolumeTempDir) throws Exception {
+ this.layoutVersion = layout;
+ OzoneConfiguration conf = perVolumeConf(perVolumeTempDir, 1);
+ MutableVolumeSet volumeSet = newVolumeSet(conf);
+ ReplicationServer.ReplicationConfig repConf =
+ conf.getObject(ReplicationServer.ReplicationConfig.class);
+
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .containerSet(set)
+ .volumeSet(volumeSet)
+ .executor(newDirectExecutorService())
+ .clock(clock)
+ .build();
+
+ try {
+ supervisor.setPerVolumePoolSize(3);
+ VolumeReplicationThreadPools pools =
+ supervisor.getVolumeReplicationThreadPools();
+ for (StorageVolume volume : volumeSet.getVolumesList()) {
+ assertEquals(3, pools.getPoolSize(volume.getStorageDir().getPath()));
+ }
+ assertEquals(3, repConf.getPerVolumeStreamsLimit());
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void perVolumePoolResizeOnNodeStateChange(ContainerLayoutVersion layout,
+ @TempDir File perVolumeTempDir) throws Exception {
+ this.layoutVersion = layout;
+ OzoneConfiguration conf = perVolumeConf(perVolumeTempDir, 2);
+ MutableVolumeSet volumeSet = newVolumeSet(conf);
+ ReplicationServer.ReplicationConfig repConf =
+ conf.getObject(ReplicationServer.ReplicationConfig.class);
+
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .containerSet(set)
+ .volumeSet(volumeSet)
+ .clock(clock)
+ .build();
+
+ try {
+ datanode.setPersistedOpState(IN_SERVICE);
+ supervisor.nodeStateUpdated(
+ HddsProtos.NodeOperationalState.DECOMMISSIONING);
+ VolumeReplicationThreadPools pools =
+ supervisor.getVolumeReplicationThreadPools();
+ int expected = repConf.scaleOutOfServiceLimit(2);
+ for (StorageVolume volume : volumeSet.getVolumesList()) {
+ assertEquals(expected,
+ pools.getPoolSize(volume.getStorageDir().getPath()));
+ }
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void perVolumePoolResizeDuringDecommission(ContainerLayoutVersion layout,
+ @TempDir File perVolumeTempDir) throws Exception {
+ this.layoutVersion = layout;
+ OzoneConfiguration conf = perVolumeConf(perVolumeTempDir, 2);
+ MutableVolumeSet volumeSet = newVolumeSet(conf);
+ ReplicationServer.ReplicationConfig repConf =
+ conf.getObject(ReplicationServer.ReplicationConfig.class);
+
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .containerSet(set)
+ .volumeSet(volumeSet)
+ .clock(clock)
+ .build();
+
+ try {
+ datanode.setPersistedOpState(IN_SERVICE);
+ supervisor.nodeStateUpdated(DECOMMISSIONING);
+ supervisor.setPerVolumePoolSize(2);
+ VolumeReplicationThreadPools pools =
+ supervisor.getVolumeReplicationThreadPools();
+ int expected = repConf.scaleOutOfServiceLimit(2);
+ for (StorageVolume volume : volumeSet.getVolumesList()) {
+ assertEquals(expected,
+ pools.getPoolSize(volume.getStorageDir().getPath()));
+ }
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void nonPushReplicationUsesGlobalPoolWhenPerVolumeEnabled(
+ ContainerLayoutVersion layout, @TempDir File perVolumeTempDir) throws Exception {
+ this.layoutVersion = layout;
+ OzoneConfiguration conf = perVolumeConf(perVolumeTempDir, 1);
+ MutableVolumeSet volumeSet = newVolumeSet(conf);
+ ReplicationServer.ReplicationConfig repConf =
+ conf.getObject(ReplicationServer.ReplicationConfig.class);
+ AtomicInteger globalExecutions = new AtomicInteger();
+
+ ExecutorService trackingGlobal = new AbstractExecutorService() {
+ @Override
+ public void shutdown() {
+ }
+
+ @Override
+ public List<Runnable> shutdownNow() {
+ return emptyList();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return false;
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return false;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) {
+ return true;
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ globalExecutions.incrementAndGet();
+ command.run();
+ }
+ };
+
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .containerSet(set)
+ .volumeSet(volumeSet)
+ .executor(trackingGlobal)
+ .clock(clock)
+ .build();
+
+ try {
+ supervisor.addTask(createReconciliationTask(1L));
+ assertEquals(1, globalExecutions.get());
+ assertEquals(1, supervisor.getReplicationSuccessCount());
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void perVolumePushIsolation(ContainerLayoutVersion layout,
+ @TempDir File perVolumeTempDir) throws Exception {
+ this.layoutVersion = layout;
+ OzoneConfiguration conf = perVolumeConf(perVolumeTempDir, 1);
+ MutableVolumeSet volumeSet = newVolumeSet(conf);
+ HddsVolume vol1 = (HddsVolume) volumeSet.getVolumesList().get(0);
+ HddsVolume vol2 = (HddsVolume) volumeSet.getVolumesList().get(1);
+
+ addContainerOnVolume(1L, vol1, conf);
+ addContainerOnVolume(2L, vol2, conf);
+
+ ReplicationServer.ReplicationConfig repConf =
+ conf.getObject(ReplicationServer.ReplicationConfig.class);
+
+ CountDownLatch vol1Started = new CountDownLatch(1);
+ CountDownLatch vol1Release = new CountDownLatch(1);
+ ContainerReplicator volumeAwareReplicator = task -> {
+ Container<?> container = set.getContainer(task.getContainerId());
+ HddsVolume volume = container.getContainerData().getVolume();
+ if (volume == vol1) {
+ vol1Started.countDown();
+ try {
+ assertTrue(vol1Release.await(10, TimeUnit.SECONDS));
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(ie);
+ }
+ }
+ task.setStatus(DONE);
+ };
+ replicatorRef.set(volumeAwareReplicator);
+
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .containerSet(set)
+ .volumeSet(volumeSet)
+ .clock(clock)
+ .build();
+
+ try {
+ supervisor.addTask(createPushTask(1L));
+ assertTrue(vol1Started.await(10, TimeUnit.SECONDS));
+
+ supervisor.addTask(createPushTask(2L));
+ GenericTestUtils.waitFor((BooleanSupplier) () ->
+ supervisor.getReplicationSuccessCount() >= 1, 100, 10000);
+
+ assertEquals(1, supervisor.getReplicationSuccessCount());
+ vol1Release.countDown();
+ GenericTestUtils.waitFor((BooleanSupplier) () ->
+ supervisor.getReplicationSuccessCount() == 2, 100, 10000);
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void volumeFailureCleansUpQueuedTasks(ContainerLayoutVersion layout,
+ @TempDir File perVolumeTempDir) throws Exception {
+ this.layoutVersion = layout;
+ OzoneConfiguration conf = perVolumeConf(perVolumeTempDir, 1);
+ MutableVolumeSet volumeSet = newVolumeSet(conf);
+ HddsVolume vol1 = (HddsVolume) volumeSet.getVolumesList().get(0);
+ addContainerOnVolume(1L, vol1, conf);
+ addContainerOnVolume(2L, vol1, conf);
+
+ ReplicationServer.ReplicationConfig repConf =
+ conf.getObject(ReplicationServer.ReplicationConfig.class);
+
+ CountDownLatch task1Started = new CountDownLatch(1);
+ CountDownLatch task1Block = new CountDownLatch(1);
+ AtomicBoolean task1Interrupted = new AtomicBoolean();
+ replicatorRef.set(task -> {
+ if (task.getContainerId() == 1L) {
+ task1Started.countDown();
+ try {
+ task1Block.await();
+ } catch (InterruptedException ie) {
+ task1Interrupted.set(true);
+ Thread.currentThread().interrupt();
+ return;
+ }
+ }
+ task.setStatus(DONE);
+ });
+
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .containerSet(set)
+ .volumeSet(volumeSet)
+ .clock(clock)
+ .build();
+
+ String volumeRoot = vol1.getStorageDir().getPath();
+ try {
+ supervisor.addTask(createPushTask(1L));
+ assertTrue(task1Started.await(10, TimeUnit.SECONDS));
+
+ supervisor.addTask(createPushTask(2L));
+ GenericTestUtils.waitFor((BooleanSupplier) () ->
+ supervisor.getTotalInFlightReplications() == 2, 100, 5000);
+
+ volumeSet.failVolume(volumeRoot);
+ supervisor.shutdownFailedVolumePools(volumeSet);
+
+ GenericTestUtils.waitFor((BooleanSupplier) () ->
+ supervisor.getTotalInFlightReplications() == 0, 100, 5000);
+ assertTrue(task1Interrupted.get());
+ task1Block.countDown();
+
+ supervisor.addTask(createPushTask(2L));
+ GenericTestUtils.waitFor((BooleanSupplier) () ->
+ supervisor.getReplicationSuccessCount() >= 1, 100, 5000);
+ } finally {
+ task1Block.countDown();
+ supervisor.stop();
+ }
+ }
+
+ @ContainerLayoutTestInfo.ContainerTest
+ public void volumeFailureShutsDownPool(ContainerLayoutVersion layout,
+ @TempDir File perVolumeTempDir) throws Exception {
+ this.layoutVersion = layout;
+ OzoneConfiguration conf = perVolumeConf(perVolumeTempDir, 1);
+ MutableVolumeSet volumeSet = newVolumeSet(conf);
+ HddsVolume vol1 = (HddsVolume) volumeSet.getVolumesList().get(0);
+ addContainerOnVolume(1L, vol1, conf);
+
+ ReplicationServer.ReplicationConfig repConf =
+ conf.getObject(ReplicationServer.ReplicationConfig.class);
+ ReplicationSupervisor supervisor = ReplicationSupervisor.newBuilder()
+ .stateContext(context)
+ .replicationConfig(repConf)
+ .containerSet(set)
+ .volumeSet(volumeSet)
+ .executor(newDirectExecutorService())
+ .clock(clock)
+ .build();
+ replicatorRef.set(doneReplicator);
+
+ String volumeRoot = vol1.getStorageDir().getPath();
+ try {
+ VolumeReplicationThreadPools pools =
+ supervisor.getVolumeReplicationThreadPools();
+ assertTrue(pools.hasPool(volumeRoot));
+
+ volumeSet.failVolume(volumeRoot);
+ supervisor.shutdownFailedVolumePools(volumeSet);
+ assertFalse(pools.hasPool(volumeRoot));
+
+ supervisor.addTask(createPushTask(1L));
+ assertEquals(1, supervisor.getReplicationSuccessCount());
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ private OzoneConfiguration perVolumeConf(File baseDir, int perVolumeStreams) {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, baseDir.getAbsolutePath());
+ conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY,
+ baseDir.getAbsolutePath() + "/vol1,"
+ + baseDir.getAbsolutePath() + "/vol2");
+ conf.setBoolean(PER_VOLUME_ENABLED_KEY, true);
+ conf.setInt(PER_VOLUME_STREAMS_LIMIT_KEY, perVolumeStreams);
+ conf.setClass(SpaceUsageCheckFactory.Conf.configKeyForClassName(),
+ MockSpaceUsageCheckFactory.HalfTera.class,
+ SpaceUsageCheckFactory.class);
+ return conf;
+ }
+
+ private MutableVolumeSet newVolumeSet(OzoneConfiguration conf)
+ throws IOException {
+ return new MutableVolumeSet(datanode.getUuidString(), conf, null,
+ StorageVolume.VolumeType.DATA_VOLUME, null);
+ }
+
+ private void addContainerOnVolume(long containerId, HddsVolume volume,
+ OzoneConfiguration conf) {
+ KeyValueContainerData containerData = new KeyValueContainerData(containerId,
+ layoutVersion, 100L,
+ UUID.randomUUID().toString(), UUID.randomUUID().toString());
+ containerData.setVolume(volume);
+ KeyValueContainer container = new KeyValueContainer(containerData, conf);
+ assertDoesNotThrow(() -> set.addContainer(container));
+ }
+
+ private ReplicationTask createPushTask(long containerId) {
- ReplicateContainerCommand cmd = ReplicateContainerCommand.toTarget(
- containerId, MockDatanodeDetails.randomDatanodeDetails());
++ ReplicateContainerCommand cmd =
++ getReplicateContainerCommand(containerId, MockDatanodeDetails.randomDatanodeDetails());
+ cmd.setTerm(CURRENT_TERM);
+ return new ReplicationTask(cmd, replicatorRef.get());
+ }
}
diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java
index a01dba245be,00000000000..7e6494c170f
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java
@@@ -1,251 -1,0 +1,251 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.upgrade;
+
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V3;
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT;
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION;
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeActionProvider;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
+import org.apache.hadoop.ozone.container.common.DatanodeStorage;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest;
+import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.UpgradeException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.provider.Arguments;
+import org.mockito.Mockito;
+
+/**
+ * Tests for {@link DatanodeVersionManager} using on-disk {@link DatanodeStorage} under a JUnit temp directory.
+ */
+class TestDatanodeVersionManager extends AbstractComponentVersionManagerTest {
+
+ private OzoneConfiguration conf;
+
+ @TempDir
+ private Path tempDir;
+
+ private static final List<ComponentVersion> ALL_VERSIONS;
+
+ static {
+ ALL_VERSIONS = new ArrayList<>(Arrays.asList(HDDSLayoutFeature.values()));
+
+ for (HDDSVersion version : HDDSVersion.values()) {
+ // Add all defined versions after and including ZDU to get the complete version list.
+ if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.UNKNOWN_VERSION) {
+ ALL_VERSIONS.add(version);
+ }
+ }
+ }
+
+ @BeforeEach
+ public void init() {
+ conf = new OzoneConfiguration();
+ }
+
+ public static Stream<Arguments> preFinalizedVersionArgs() {
+ return ALL_VERSIONS.stream()
+ .limit(ALL_VERSIONS.size() - 1)
+ .map(Arguments::of);
+ }
+
+ @Override
+ protected DatanodeVersionManager createManager(int serializedApparentVersion) throws IOException {
+ return createManager(serializedApparentVersion, HashMap::new);
+ }
+
+ private DatanodeVersionManager createManager(int serializedApparentVersion,
+ ComponentUpgradeActionProvider<DatanodeUpgradeAction> actions) throws IOException {
+ DatanodeStorage storage = newDatanodeStorage(serializedApparentVersion);
+ return new DatanodeVersionManager(storage, null, actions);
+ }
+
+ private DatanodeStorage newDatanodeStorage(int apparentVersion) throws IOException {
+ Path storageRoot = Files.createTempDirectory(tempDir, "dn-version-manager-");
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageRoot.toString());
+ DatanodeStorage storage = new DatanodeStorage(conf, UUID.randomUUID().toString(), apparentVersion);
+ storage.setApparentVersion(apparentVersion);
+ assertTrue(storage.getCurrentDir().mkdirs());
+ storage.persistCurrentState();
+ return storage;
+ }
+
+ @Override
+ protected List<ComponentVersion> allVersionsInOrder() {
+ return ALL_VERSIONS;
+ }
+
+ @Override
+ protected ComponentVersion expectedSoftwareVersion() {
+ return HDDSVersion.SOFTWARE_VERSION;
+ }
+
+ @Override
+ @Test
+ public void testClasspathScanDiscoversUpgradeActions() throws Exception {
+ // Regardless of whether Datanode is finalized, the same set of upgrade actions should be loaded.
+ try (DatanodeVersionManager versionManager = createManager(INITIAL_VERSION.serialize(),
+ new DatanodeUpgradeActionProvider())) {
+ assertTrue(versionManager.needsFinalization());
+ DatanodeUpgradeAction quotaAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V3);
+ assertInstanceOf(DatanodeSchemaV3FinalizeAction.class, quotaAction);
+ DatanodeUpgradeAction zduAction = versionManager.getUpgradeActionsForTesting().get(HDDSVersion.ZDU);
+ assertInstanceOf(ZduDatanodeUpgradeActionForTest.class, zduAction);
+ }
+
+ try (DatanodeVersionManager versionManager =
+ createManager(HDDSVersion.SOFTWARE_VERSION.serialize(), new DatanodeUpgradeActionProvider())) {
+ assertFalse(versionManager.needsFinalization());
+ DatanodeUpgradeAction quotaAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V3);
+ assertInstanceOf(DatanodeSchemaV3FinalizeAction.class, quotaAction);
+ DatanodeUpgradeAction zduAction = versionManager.getUpgradeActionsForTesting().get(HDDSVersion.ZDU);
+ assertInstanceOf(ZduDatanodeUpgradeActionForTest.class, zduAction);
+ }
+ }
+
+ @Override
+ @Test
+ public void testFinalizeRunsSuppliedUpgradeAction() throws Exception {
+ DatanodeUpgradeAction mockECAction = mock(DatanodeUpgradeAction.class);
+ DatanodeUpgradeAction mockZDUAction = mock(DatanodeUpgradeAction.class);
+
+ ComponentUpgradeActionProvider<DatanodeUpgradeAction> provider = () -> {
+ Map<ComponentVersion, DatanodeUpgradeAction> m = new HashMap<>();
+ m.put(ERASURE_CODED_STORAGE_SUPPORT, mockECAction);
+ m.put(HDDSVersion.ZDU, mockZDUAction);
+ return m;
+ };
+
+ try (DatanodeVersionManager versionManager = createManager(DATANODE_SCHEMA_V3.serialize(), provider)) {
+ versionManager.finalizeUpgrade();
+ assertEquals(HDDSVersion.SOFTWARE_VERSION, versionManager.getApparentVersion());
+
+ // DATANODE_SCHEMA_V3 was added after EC, so the EC upgrade action should not run when we finalize from this
+ // version.
+ verify(mockECAction, never()).execute(any());
+ verify(mockZDUAction, atLeastOnce()).execute(any());
+ assertDatanodeApparentVersionOnDisk(conf, HDDSVersion.SOFTWARE_VERSION.serialize());
+ }
+ }
+
+ @Override
+ @Test
+ public void testUpgradeActionFailureAbortsFinalize() throws Exception {
+ ComponentUpgradeActionProvider<DatanodeUpgradeAction> provider = () -> {
+ Map<ComponentVersion, DatanodeUpgradeAction> m = new HashMap<>();
+ m.put(STORAGE_SPACE_DISTRIBUTION, o -> {
+ throw new IOException("expected test failure");
+ });
+ return m;
+ };
+
+ try (DatanodeVersionManager versionManager =
+ createManager(WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize(), provider)) {
+ UpgradeException thrown = assertThrows(UpgradeException.class, versionManager::finalizeUpgrade);
+ assertEquals(UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED, thrown.getResult());
+ // WITNESSED_CONTAINER_DB_PROTO_VALUE is the version before STORAGE_SPACE_DISTRIBUTION, which has failed.
+ assertEquals(WITNESSED_CONTAINER_DB_PROTO_VALUE, versionManager.getApparentVersion());
+ assertDatanodeApparentVersionOnDisk(conf, WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize());
+ }
+ }
+
+ @Override
+ @Test
+ public void testPersistFailureRollsBack() throws Exception {
+ // Create a mock storage instance that throws when persisting version updates.
+ DatanodeStorage storage = mock(DatanodeStorage.class);
+ AtomicInteger persistedApparentVersion = new AtomicInteger(INITIAL_VERSION.serialize());
+ when(storage.getApparentVersion()).thenAnswer(invocation -> persistedApparentVersion.get());
+ doAnswer(invocation -> {
+ persistedApparentVersion.set(invocation.getArgument(0));
+ return null;
+ }).when(storage).setApparentVersion(anyInt());
+ doThrow(new IOException("persist failed")).when(storage).persistCurrentState();
+
+ DatanodeStateMachine mockDN = Mockito.mock(DatanodeStateMachine.class);
+ try (DatanodeVersionManager versionManager = new DatanodeVersionManager(storage, mockDN, HashMap::new)) {
+ assertEquals(INITIAL_VERSION, versionManager.getApparentVersion());
+ UpgradeException thrown = assertThrows(UpgradeException.class, versionManager::finalizeUpgrade);
+ assertEquals(UpgradeException.ResultCodes.APPARENT_VERSION_UPDATE_FAILED, thrown.getResult());
+ assertEquals(INITIAL_VERSION, versionManager.getApparentVersion());
+ assertEquals(INITIAL_VERSION.serialize(), versionManager.getPersistedApparentVersion());
+ }
+ }
+
+ @Test
+ public void testGetVersionForClientWhenFinalized() throws Exception {
+ // Apparent version >= ZDU (finalized) reports the apparent version itself.
+ try (DatanodeVersionManager versionManager = createManager(HDDSVersion.ZDU.serialize())) {
+ assertEquals(HDDSVersion.ZDU, versionManager.getVersionForClient());
+ }
+
+ try (DatanodeVersionManager versionManager = createManager(HDDSVersion.SOFTWARE_VERSION.serialize())) {
+ assertEquals(HDDSVersion.SOFTWARE_VERSION, versionManager.getVersionForClient());
+ }
+ }
+
+ @Test
+ public void testGetVersionForClientWhenPreZdu() throws Exception {
+ // A pre-ZDU apparent version clamps to the last pre-ZDU wire version.
+ try (DatanodeVersionManager versionManager = createManager(INITIAL_VERSION.serialize())) {
- assertEquals(HDDSVersion.STREAM_BLOCK_SUPPORT, versionManager.getVersionForClient());
++ assertEquals(HDDSVersion.SHORT_CIRCUIT_READS, versionManager.getVersionForClient());
+ }
+ }
+
+ private static void assertDatanodeApparentVersionOnDisk(OzoneConfiguration conf, int expected)
+ throws IOException {
+ DatanodeStorage reloaded = new DatanodeStorage(conf, UUID.randomUUID().toString());
+ assertEquals(expected, reloaded.getApparentVersion());
+ }
+}
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 9f4f5353a77,fe886bf0c20..eca5933a2aa
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@@ -470,7 -473,12 +470,12 @@@ public class SCMNodeManager implements
LOG.info("Update the version for registered datanode {}, " +
"oldVersion = {}, newVersion = {}.",
datanodeDetails, oldNode.getVersion(), datanodeDetails.getVersion());
- nodeStateManager.updateNode(datanodeDetails, layoutInfo);
+ nodeStateManager.updateNode(datanodeDetails, dnVersionInfo);
+ } else if (oldNode.portsChanged(datanodeDetails)) {
+ // Refresh the stored node when its port set changes
+ LOG.info("Updating ports for registered datanode {}: {} -> {}",
+ datanodeDetails, oldNode.getPorts(), datanodeDetails.getPorts());
- nodeStateManager.updateNode(datanodeDetails, layoutInfo);
++ nodeStateManager.updateNode(datanodeDetails, dnVersionInfo);
}
} catch (NodeNotFoundException e) {
LOG.error("Cannot find datanode {} from nodeStateManager",
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
index eea8569b335,78814b53e81..f247cfb0e40
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@@ -64,6 -62,9 +64,7 @@@ import java.util.Arrays
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+ import java.util.UUID;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@@ -287,6 -346,271 +288,81 @@@ public class TestSCMNodeManager
return cmd.getDatanode();
}
+ private static DatanodeDetails.Builder datanodeWithoutDatastream(UUID uuid) {
+ return DatanodeDetails.newBuilder()
+ .setUuid(uuid)
+ .setHostName("host-" + uuid)
+ .setIpAddress("127.0.0.1")
+ .addPort(DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.STANDALONE, 9859))
+ .addPort(DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.RATIS, 9858));
+ }
+
+ private void registerNode(SCMNodeManager nodeManager, DatanodeDetails dn) {
+ StorageReportProto storageReport = HddsTestUtils.createStorageReport(
+ dn.getID(), dn.getNetworkFullPath(), Long.MAX_VALUE);
+ MetadataStorageReportProto metadataStorageReport =
+ HddsTestUtils.createMetadataStorageReport(
+ dn.getNetworkFullPath(), Long.MAX_VALUE);
+ RegisteredCommand cmd = nodeManager.register(dn,
+ HddsTestUtils.createNodeReport(Arrays.asList(storageReport),
+ Arrays.asList(metadataStorageReport)),
- getRandomPipelineReports(), UpgradeUtils.defaultLayoutVersionProto());
++ getRandomPipelineReports(), UpgradeUtils.defaultVersionProto());
+ assertEquals(success, cmd.getError());
+ }
+
+ /**
+ * A datanode that re-registers with the same identity but now exposes the
+ * RATIS_DATASTREAM port (e.g. Ratis DataStream was enabled) must have its
+ * stored record refreshed so the new port is visible (HDDS-15799).
+ */
+ @Test
+ public void testRegisterRefreshesPortsOnPortChange()
+ throws IOException, AuthenticationException {
+ try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
+ final UUID uuid = UUID.randomUUID();
+
+ // First registration: streaming disabled, no RATIS_DATASTREAM port.
+ registerNode(nodeManager, datanodeWithoutDatastream(uuid).build());
+ DatanodeDetails stored = nodeManager.getNode(
+ datanodeWithoutDatastream(uuid).build().getID());
+ assertFalse(stored.hasPort(DatanodeDetails.Port.Name.RATIS_DATASTREAM));
+
+ // Re-registration (same id/ip/host/version) now exposing the port.
+ registerNode(nodeManager, datanodeWithoutDatastream(uuid)
+ .addPort(DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.RATIS_DATASTREAM, 9855))
+ .build());
+ stored = nodeManager.getNode(
+ datanodeWithoutDatastream(uuid).build().getID());
+ assertTrue(stored.hasPort(DatanodeDetails.Port.Name.RATIS_DATASTREAM),
+ "stored node should be refreshed with the RATIS_DATASTREAM port");
+ }
+ }
+
+ /**
+ * Re-registering a datanode with an unchanged port set must not disturb the
+ * stored record (the port-refresh branch is skipped).
+ */
+ @Test
+ public void testRegisterKeepsPortsWhenUnchanged()
+ throws IOException, AuthenticationException {
+ try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
+ final UUID uuid = UUID.randomUUID();
+
+ registerNode(nodeManager, datanodeWithoutDatastream(uuid).build());
+ // Re-register with the identical port set.
+ registerNode(nodeManager, datanodeWithoutDatastream(uuid).build());
+
+ final DatanodeDetails stored = nodeManager.getNode(
+ datanodeWithoutDatastream(uuid).build().getID());
+ assertTrue(stored.hasPort(DatanodeDetails.Port.Name.STANDALONE));
+ assertTrue(stored.hasPort(DatanodeDetails.Port.Name.RATIS));
+ assertFalse(stored.hasPort(DatanodeDetails.Port.Name.RATIS_DATASTREAM));
+ }
+ }
+
- private void assertPipelineClosedAfterLayoutHeartbeat(
- DatanodeDetails originalNode1, DatanodeDetails originalNode2,
- SCMNodeManager nodeManager, LayoutVersionProto layout) throws Exception {
-
- List<DatanodeDetails> originalNodes =
- Arrays.asList(originalNode1, originalNode2);
-
- // Initial condition: 2 healthy nodes registered.
- assertPipelines(HddsProtos.ReplicationFactor.ONE, count -> count == 2,
- originalNodes);
- assertPipelines(HddsProtos.ReplicationFactor.THREE,
- count -> count == 0, new ArrayList<>());
-
- // Even when safemode exit or new node addition trigger pipeline
- // creation, they will fail with not enough healthy nodes for ratis 3
- // pipeline. Therefore we do not have to worry about this create call
- // failing due to datanodes reaching their maximum pipeline limit.
- assertPipelineCreationFailsWithNotEnoughNodes(2);
-
- // Register a new node correctly.
- DatanodeDetails node = registerWithCapacity(nodeManager);
-
- List<DatanodeDetails> allNodes = new ArrayList<>(originalNodes);
- allNodes.add(node);
-
- // Safemode exit and adding the new node should trigger pipeline creation.
- assertPipelines(HddsProtos.ReplicationFactor.ONE, count -> count == 3,
- allNodes);
- assertPipelines(HddsProtos.ReplicationFactor.THREE, count -> count >= 1,
- allNodes);
-
- // node sends incorrect layout.
- nodeManager.processLayoutVersionReport(node, layout);
-
- // Its pipelines should be closed then removed, meaning there is not
- // enough nodes for factor 3 pipelines.
- assertPipelines(HddsProtos.ReplicationFactor.ONE, count -> count == 2,
- originalNodes);
- assertPipelines(HddsProtos.ReplicationFactor.THREE,
- count -> count == 0, new ArrayList<>());
-
- assertPipelineCreationFailsWithNotEnoughNodes(2);
- }
-
- /**
- * Tests that node manager handles layout versions for newly registered nodes
- * correctly.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- public void testScmLayoutOnRegister()
- throws Exception {
-
- OzoneConfiguration conf = getConf();
- conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL,
- 1, TimeUnit.DAYS);
-
- try (SCMNodeManager nodeManager = createNodeManager(conf)) {
- assertTrue(scm.getScmContext().isLeader());
- // Nodes with mismatched SLV cannot join the cluster.
- registerWithCapacity(nodeManager,
- LARGER_SLV_LAYOUT_PROTO, errorNodeNotPermitted);
- registerWithCapacity(nodeManager,
- SMALLER_MLV_SLV_LAYOUT_PROTO, errorNodeNotPermitted);
- registerWithCapacity(nodeManager,
- LARGER_MLV_SLV_LAYOUT_PROTO, errorNodeNotPermitted);
- // Nodes with mismatched MLV can join, but should not be allowed in
- // pipelines.
- DatanodeDetails badMlvNode1 = registerWithCapacity(nodeManager,
- SMALLER_MLV_LAYOUT_PROTO, success);
- DatanodeDetails badMlvNode2 = registerWithCapacity(nodeManager,
- SMALLER_MLV_LAYOUT_PROTO, success);
- // This node has correct MLV and SLV, so it can join and be used in
- // pipelines.
- DatanodeDetails goodNode = registerWithCapacity(nodeManager,
- CORRECT_LAYOUT_PROTO, success);
-
- assertEquals(3, nodeManager.getAllNodes().size());
-
- scm.exitSafeMode();
-
- // SCM should auto create a factor 1 pipeline for the one healthy node.
- // Still should not have enough healthy nodes for ratis 3 pipeline.
- assertPipelines(HddsProtos.ReplicationFactor.ONE,
- count -> count == 1,
- Collections.singletonList(goodNode));
- assertPipelines(HddsProtos.ReplicationFactor.THREE,
- count -> count == 0,
- new ArrayList<>());
-
- // Even when safemode exit or new node addition trigger pipeline
- // creation, they will fail with not enough healthy nodes for ratis 3
- // pipeline. Therefore we do not have to worry about this create call
- // failing due to datanodes reaching their maximum pipeline limit.
- assertPipelineCreationFailsWithExceedingLimit(2);
-
- // Heartbeat bad MLV nodes back to healthy.
- nodeManager.processLayoutVersionReport(badMlvNode1, CORRECT_LAYOUT_PROTO);
- nodeManager.processLayoutVersionReport(badMlvNode2, CORRECT_LAYOUT_PROTO);
- nodeManager.processHeartbeat(badMlvNode1);
- nodeManager.processHeartbeat(badMlvNode2);
-
- // After moving out of healthy readonly, pipeline creation should be
- // triggered.
- assertPipelines(HddsProtos.ReplicationFactor.ONE,
- count -> count == 3,
- Arrays.asList(badMlvNode1, badMlvNode2, goodNode));
- assertPipelines(HddsProtos.ReplicationFactor.THREE,
- count -> count >= 1,
- Arrays.asList(badMlvNode1, badMlvNode2, goodNode));
- }
- }
-
- private void assertPipelineCreationFailsWithNotEnoughNodes(
- int actualNodeCount) throws Exception {
- SCMException ex = assertThrows(SCMException.class, () -> {
- ReplicationConfig ratisThree =
- ReplicationConfig.fromProtoTypeAndFactor(
- HddsProtos.ReplicationType.RATIS,
- HddsProtos.ReplicationFactor.THREE);
- scm.getPipelineManager().createPipeline(ratisThree);
- }, "3 nodes should not have been found for a pipeline.");
- assertThat(ex.getMessage()).contains("Required 3. Found " +
- actualNodeCount);
- }
-
- private void assertPipelineCreationFailsWithExceedingLimit(int limit) {
- // Build once, outside the assertion
- ReplicationConfig config = ReplicationConfig.fromProtoTypeAndFactor(
- HddsProtos.ReplicationType.RATIS,
- HddsProtos.ReplicationFactor.THREE);
- SCMException ex = assertThrows(
- SCMException.class,
- () -> scm.getPipelineManager().createPipeline(config),
- "3 nodes should not have been found for a pipeline.");
- assertThat(ex.getMessage())
- .contains("Cannot create pipeline as it would exceed the limit per datanode: " + limit);
- }
-
- private void assertPipelines(HddsProtos.ReplicationFactor factor,
- Predicate<Integer> countCheck, Collection<DatanodeDetails> allowedDNs)
- throws Exception {
-
- Set<String> allowedDnIds = allowedDNs.stream()
- .map(DatanodeDetails::getUuidString)
- .collect(Collectors.toSet());
-
- RatisReplicationConfig replConfig = RatisReplicationConfig
- .getInstance(factor);
-
- // Wait for the expected number of pipelines using allowed DNs.
- GenericTestUtils.waitFor(() -> {
- // Closed pipelines are no longer in operation so we should not count
- // them. We cannot check for open pipelines only because this is a mock
- // test so the pipelines may remain in ALLOCATED state.
- List<Pipeline> pipelines = scm.getPipelineManager()
- .getPipelines(replConfig)
- .stream()
- .filter(p -> p.getPipelineState() != Pipeline.PipelineState.CLOSED)
- .collect(Collectors.toList());
- LOG.info("Found {} non-closed pipelines of type {} and factor {}.",
- pipelines.size(),
- replConfig.getReplicationType(), replConfig.getReplicationFactor());
- boolean success = countCheck.test(pipelines.size());
-
- // If we have the correct number of pipelines, make sure that none of
- // these pipelines use nodes outside of allowedDNs.
- if (success) {
- for (Pipeline pipeline: pipelines) {
- for (DatanodeDetails pipelineDN: pipeline.getNodes()) {
- // Do not wait for this condition to be true. Disallowed DNs should
- // never be used once we have the expected number of pipelines.
- if (!allowedDnIds.contains(pipelineDN.getUuidString())) {
- String message = String.format("Pipeline %s used datanode %s " +
- "which is not in the set of allowed datanodes: %s",
- pipeline.getId().toString(), pipelineDN.getUuidString(),
- allowedDnIds);
- fail(message);
- }
- }
- }
- }
-
- return success;
- }, 1000, 10000);
- }
-
/**
* asserts that if we send no heartbeats node manager stays in safemode.
*
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
index 2f1d5ede5d7,e980b81318d..3e46dd24380
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
@@@ -100,7 -102,8 +102,9 @@@ import org.apache.hadoop.hdds.utils.db.
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+ import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
++import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
import org.apache.ozone.test.MockClock;
@@@ -980,4 -983,187 +984,187 @@@ public class TestPipelineManagerImpl
assertEquals(ResultCodes.SCM_NOT_LEADER, e.getResult());
assertInstanceOf(NotLeaderException.class, e.getCause());
}
+
+ private static DatanodeDetails portlessDatanode(DatanodeID id) {
+ return DatanodeDetails.newBuilder()
+ .setID(id)
+ .setHostName("host-" + id)
+ .setIpAddress("127.0.0.1")
+ .addPort(DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.STANDALONE, 9859))
+ .addPort(DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.RATIS, 9858))
+ .build();
+ }
+
+ private Pipeline addPipeline(PipelineManagerImpl pipelineManager,
+ Pipeline.PipelineState state, List<DatanodeDetails> nodes)
+ throws IOException {
+ final Pipeline pipeline = Pipeline.newBuilder()
+ .setReplicationConfig(
+ RatisReplicationConfig.getInstance(ReplicationFactor.THREE))
+ .setNodes(nodes)
+ .setState(state)
+ .setId(PipelineID.randomId())
+ .build();
+ pipelineManager.getStateManager().addPipeline(
- pipeline.getProtobufMessage(ClientVersion.CURRENT_VERSION));
++ pipeline.getProtobufMessage(ClientVersion.CURRENT));
+ return pipeline;
+ }
+
+ private static boolean exists(PipelineManagerImpl pipelineManager,
+ PipelineID id) {
+ try {
+ pipelineManager.getPipeline(id);
+ return true;
+ } catch (PipelineNotFoundException e) {
+ return false;
+ }
+ }
+
+ @Test
+ public void testClosePipelinesExposingNewPorts() throws Exception {
+ conf.setBoolean(HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, true);
+ try (PipelineManagerImpl pipelineManager = createPipelineManager(true)) {
+ // Registered datanodes (MockNodeManager) expose all ports incl datastream.
+ final List<DatanodeInfo> registered = nodeManager.getAllNodes();
+ final List<DatanodeDetails> idsA = new ArrayList<>();
+ final List<DatanodeDetails> idsB = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ idsA.add(portlessDatanode(registered.get(i).getID()));
+ idsB.add(portlessDatanode(registered.get(i + 3).getID()));
+ }
+
+ // OPEN, registered nodes, portless -> legacy pipeline, must be closed.
+ final Pipeline stale = addPipeline(pipelineManager, OPEN, idsA);
+ // OPEN, registered nodes carrying all ports -> not stale, kept.
+ final Pipeline portful = addPipeline(pipelineManager, OPEN,
+ new ArrayList<>(registered.subList(6, 9)));
+ // OPEN, but nodes are NOT registered -> cannot heal, left alone.
+ final List<DatanodeDetails> unregistered = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ unregistered.add(portlessDatanode(DatanodeID.randomID()));
+ }
+ final Pipeline unreg = addPipeline(pipelineManager, OPEN, unregistered);
+ // ALLOCATED (non-open) portless -> skipped.
+ final Pipeline allocated = addPipeline(pipelineManager, ALLOCATED, idsB);
+
+ pipelineManager.closePipelinesMissingDataStreamPort();
+
+ assertFalse(exists(pipelineManager, stale.getId()),
+ "OPEN pipeline whose nodes expose new ports should be closed and deleted");
+ assertTrue(exists(pipelineManager, portful.getId()));
+ assertTrue(exists(pipelineManager, unreg.getId()));
+ assertTrue(exists(pipelineManager, allocated.getId()));
+ }
+ }
+
+ @Test
+ public void testClosePipelinesExposingNewPortsSkippedWhenDataStreamDisabled()
+ throws Exception {
+ // Datastream disabled (default): even a portless RATIS pipeline is kept.
+ try (PipelineManagerImpl pipelineManager = createPipelineManager(true)) {
+ final List<DatanodeInfo> registered = nodeManager.getAllNodes();
+ final List<DatanodeDetails> nodes = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ nodes.add(portlessDatanode(registered.get(i).getID()));
+ }
+ final Pipeline portless = addPipeline(pipelineManager, OPEN, nodes);
+
+ pipelineManager.closePipelinesMissingDataStreamPort();
+
+ assertTrue(exists(pipelineManager, portless.getId()),
+ "portless pipeline must be kept while datastream is disabled");
+ }
+ }
+
+ @Test
+ public void testClosePipelinesExposingNewPortsSkipsEcPipeline()
+ throws Exception {
+ conf.setBoolean(HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, true);
+ try (PipelineManagerImpl pipelineManager = createPipelineManager(true)) {
+ final List<DatanodeInfo> registered = nodeManager.getAllNodes();
+ final List<DatanodeDetails> nodes = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ nodes.add(portlessDatanode(registered.get(i).getID()));
+ }
+ final Pipeline ec = Pipeline.newBuilder()
+ .setReplicationConfig(new ECReplicationConfig(3, 2))
+ .setNodes(nodes)
+ .setState(OPEN)
+ .setId(PipelineID.randomId())
+ .build();
+ pipelineManager.getStateManager().addPipeline(
- ec.getProtobufMessage(ClientVersion.CURRENT_VERSION));
++ ec.getProtobufMessage(ClientVersion.CURRENT));
+
+ pipelineManager.closePipelinesMissingDataStreamPort();
+
+ assertTrue(exists(pipelineManager, ec.getId()),
+ "EC pipeline must not be closed by datastream port scrubbing");
+ }
+ }
+
+ @Test
+ public void testClosePipelinesExposingNewPortsKeepsNotYetRestartedNodes()
+ throws Exception {
+ conf.setBoolean(HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, true);
+ try (PipelineManagerImpl pipelineManager = createPipelineManager(true)) {
+ // Nodes are registered and healthy but still lack the datastream port
+ // (they have not restarted yet during a rolling enablement).
+ final List<DatanodeDetails> nodes = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ final DatanodeDetails portless = portlessDatanode(DatanodeID.randomID());
+ nodeManager.register(new DatanodeInfo(portless,
- NodeStatus.inServiceHealthy(), null,
++ NodeStatus.inServiceHealthy(), UpgradeUtils.defaultVersionProto(),
+ HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT), null, null);
+ nodes.add(portless);
+ }
+ final Pipeline pending = addPipeline(pipelineManager, OPEN, nodes);
+
+ pipelineManager.closePipelinesMissingDataStreamPort();
+
+ assertTrue(exists(pipelineManager, pending.getId()),
+ "pipeline whose registered nodes have not yet advertised the "
+ + "datastream port must be kept");
+ }
+ }
+
+ @Test
+ public void testClosePipelinesExposingNewPortsSwallowsError() throws Exception {
+ conf.setBoolean(HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, true);
+ try (PipelineManagerImpl pipelineManager = createPipelineManager(true)) {
+ final List<DatanodeInfo> registered = nodeManager.getAllNodes();
+ final List<DatanodeDetails> nodes = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ nodes.add(portlessDatanode(registered.get(i).getID()));
+ }
+ final Pipeline stale = addPipeline(pipelineManager, OPEN, nodes);
+
+ final PipelineManagerImpl spy = spy(pipelineManager);
+ doThrow(new IOException("boom")).when(spy).closePipeline(stale.getId());
+ // The close failure is logged and swallowed; the loop does not throw.
+ spy.closePipelinesMissingDataStreamPort();
+ assertTrue(exists(pipelineManager, stale.getId()));
+ }
+ }
+
+ @Test
+ public void testScrubAndCloseWiring() throws Exception {
+ // The background task scrubs then closes pipelines exposing new ports; on
+ // an empty manager both are no-ops and must not throw.
+ try (PipelineManagerImpl pipelineManager = createPipelineManager(true)) {
+ pipelineManager.scrubAndClosePipelinesMissingDataStreamPort();
+ }
+ }
+
+ @Test
+ public void testScrubAndCloseSwallowsScrubError() throws Exception {
+ try (PipelineManagerImpl pipelineManager = createPipelineManager(true)) {
+ final PipelineManagerImpl spy = spy(pipelineManager);
+ doThrow(new IOException("boom")).when(spy).scrubPipelines();
+ // Scrub failure is logged and swallowed; the close pass still runs.
+ spy.scrubAndClosePipelinesMissingDataStreamPort();
+ verify(spy).closePipelinesMissingDataStreamPort();
+ }
+ }
}
diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/upgrade/TestScmVersionManager.java
index bb8c0ddde95,00000000000..364a9310e9f
mode 100644,000000..100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/upgrade/TestScmVersionManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/upgrade/TestScmVersionManager.java
@@@ -1,248 -1,0 +1,248 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.server.upgrade;
+
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATANODE_SCHEMA_V2;
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT;
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION;
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION;
+import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.WITNESSED_CONTAINER_DB_PROTO_VALUE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
+import org.apache.hadoop.hdds.upgrade.ScmUpgradeAction;
+import org.apache.hadoop.hdds.upgrade.ScmUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest;
+import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider;
+import org.apache.hadoop.ozone.upgrade.UpgradeException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.provider.Arguments;
+
+/**
+ * Tests for {@link ScmVersionManager} using on-disk {@link SCMStorageConfig}
+ * under a JUnit temp directory.
+ */
+class TestScmVersionManager extends AbstractComponentVersionManagerTest {
+
+ private OzoneConfiguration conf;
+
+ @TempDir
+ private Path tempDir;
+
+ private static final List<ComponentVersion> ALL_VERSIONS;
+
+ static {
+ ALL_VERSIONS = new ArrayList<>(Arrays.asList(HDDSLayoutFeature.values()));
+ for (HDDSVersion version : HDDSVersion.values()) {
+ // Add all defined versions after and including ZDU to get the complete version list.
+ if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.UNKNOWN_VERSION) {
+ ALL_VERSIONS.add(version);
+ }
+ }
+ }
+
+ @BeforeEach
+ public void init() {
+ conf = new OzoneConfiguration();
+ }
+
+ public static Stream<Arguments> preFinalizedVersionArgs() {
+ return ALL_VERSIONS.stream()
+ .limit(ALL_VERSIONS.size() - 1)
+ .map(org.junit.jupiter.params.provider.Arguments::of);
+ }
+
+ @Override
+ protected ScmVersionManager createManager(int serializedApparentVersion) throws IOException {
+ return createManager(serializedApparentVersion, HashMap::new);
+ }
+
+ private ScmVersionManager createManager(int serializedApparentVersion,
+ ComponentUpgradeActionProvider<ScmUpgradeAction> actions) throws IOException {
+ SCMStorageConfig storage = newScmStorage(serializedApparentVersion);
+ StorageContainerManager context = mock(StorageContainerManager.class);
+ return new ScmVersionManager(storage, context, actions);
+ }
+
+ private SCMStorageConfig newScmStorage(int apparentVersion) throws IOException {
+ Path storageRoot = Files.createTempDirectory(tempDir, "scm-version-manager-");
+ conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, storageRoot.toString());
+ SCMStorageConfig storage = new SCMStorageConfig(conf);
+ storage.setScmId("test-scm");
+ storage.setApparentVersion(apparentVersion);
+ storage.initialize();
+ return storage;
+ }
+
+ @Override
+ protected List<ComponentVersion> allVersionsInOrder() {
+ return ALL_VERSIONS;
+ }
+
+ @Override
+ protected ComponentVersion expectedSoftwareVersion() {
+ return HDDSVersion.SOFTWARE_VERSION;
+ }
+
+ @Override
+ @Test
+ public void testClasspathScanDiscoversUpgradeActions() throws Exception {
+ try (ScmVersionManager versionManager = createManager(INITIAL_VERSION.serialize(),
+ new ScmUpgradeActionProvider())) {
+ assertTrue(versionManager.needsFinalization());
+ ScmUpgradeAction upgradeAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V2);
+ assertInstanceOf(ScmOnFinalizeActionForDatanodeSchemaV2.class, upgradeAction);
+ ScmUpgradeAction zduAction = versionManager.getUpgradeActionsForTesting().get(HDDSVersion.ZDU);
+ assertInstanceOf(ClearFinalizingStateScmUpgradeAction.class, zduAction);
+ }
+
+ try (ScmVersionManager versionManager = createManager(HDDSVersion.SOFTWARE_VERSION.serialize(),
+ new ScmUpgradeActionProvider())) {
+ assertFalse(versionManager.needsFinalization());
+ ScmUpgradeAction upgradeAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V2);
+ assertInstanceOf(ScmOnFinalizeActionForDatanodeSchemaV2.class, upgradeAction);
+ ScmUpgradeAction zduAction = versionManager.getUpgradeActionsForTesting().get(HDDSVersion.ZDU);
+ assertInstanceOf(ClearFinalizingStateScmUpgradeAction.class, zduAction);
+ }
+ }
+
+ @Override
+ @Test
+ public void testFinalizeRunsSuppliedUpgradeAction() throws Exception {
+ ScmUpgradeAction mockECAction = mock(ScmUpgradeAction.class);
+ ScmUpgradeAction mockZDUAction = mock(ScmUpgradeAction.class);
+
+ ComponentUpgradeActionProvider<ScmUpgradeAction> provider = () -> {
+ Map<ComponentVersion, ScmUpgradeAction> m = new HashMap<>();
+ m.put(ERASURE_CODED_STORAGE_SUPPORT, mockECAction);
+ m.put(HDDSVersion.ZDU, mockZDUAction);
+ return m;
+ };
+
+ try (ScmVersionManager versionManager = createManager(ERASURE_CODED_STORAGE_SUPPORT.serialize(), provider)) {
+ versionManager.finalizeUpgrade();
+ assertEquals(HDDSVersion.SOFTWARE_VERSION, versionManager.getApparentVersion());
+
+ // Apparent version is already EC; finalization runs actions for later versions only, not for EC itself.
+ verify(mockECAction, never()).execute(any());
+ verify(mockZDUAction, atLeastOnce()).execute(any());
+ assertScmApparentVersionOnDisk(conf, HDDSVersion.SOFTWARE_VERSION.serialize());
+ }
+ }
+
+ @Override
+ @Test
+ public void testUpgradeActionFailureAbortsFinalize() throws Exception {
+ ComponentUpgradeActionProvider<ScmUpgradeAction> provider = () -> {
+ Map<ComponentVersion, ScmUpgradeAction> m = new HashMap<>();
+ m.put(STORAGE_SPACE_DISTRIBUTION, o -> {
+ throw new IOException("expected test failure");
+ });
+ return m;
+ };
+
+ try (ScmVersionManager versionManager =
+ createManager(WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize(), provider)) {
+ UpgradeException thrown = assertThrows(UpgradeException.class, versionManager::finalizeUpgrade);
+ assertEquals(UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED, thrown.getResult());
+ // WITNESSED_CONTAINER_DB_PROTO_VALUE is the version before STORAGE_SPACE_DISTRIBUTION, which has failed.
+ assertEquals(WITNESSED_CONTAINER_DB_PROTO_VALUE, versionManager.getApparentVersion());
+ assertScmApparentVersionOnDisk(conf, WITNESSED_CONTAINER_DB_PROTO_VALUE.serialize());
+ }
+ }
+
+ @Override
+ @Test
+ public void testPersistFailureRollsBack() throws Exception {
+ SCMStorageConfig storage = mock(SCMStorageConfig.class);
+ AtomicInteger persistedApparentVersion = new AtomicInteger(INITIAL_VERSION.serialize());
+ when(storage.getApparentVersion()).thenAnswer(invocation -> persistedApparentVersion.get());
+ doAnswer(invocation -> {
+ persistedApparentVersion.set(invocation.getArgument(0));
+ return null;
+ }).when(storage).setApparentVersion(anyInt());
+ doThrow(new IOException("persist failed")).when(storage).persistCurrentState();
+
+ StorageContainerManager context = mock(StorageContainerManager.class);
+ try (ScmVersionManager versionManager = new ScmVersionManager(storage, context, HashMap::new)) {
+ assertEquals(INITIAL_VERSION, versionManager.getApparentVersion());
+ UpgradeException thrown = assertThrows(UpgradeException.class, versionManager::finalizeUpgrade);
+ assertEquals(UpgradeException.ResultCodes.APPARENT_VERSION_UPDATE_FAILED, thrown.getResult());
+ assertEquals(INITIAL_VERSION, versionManager.getApparentVersion());
+ assertEquals(INITIAL_VERSION.serialize(), versionManager.getPersistedApparentVersion());
+ assertEquals(INITIAL_VERSION.serialize(), storage.getApparentVersion());
+ }
+ }
+
+ @Test
+ public void testGetVersionForClientWhenFinalized() throws Exception {
+ // Apparent version >= ZDU (finalized) reports the apparent version itself.
+ try (ScmVersionManager versionManager = createManager(HDDSVersion.ZDU.serialize())) {
+ assertEquals(HDDSVersion.ZDU, versionManager.getVersionForClient());
+ }
+
+ try (ScmVersionManager versionManager = createManager(HDDSVersion.SOFTWARE_VERSION.serialize())) {
+ assertEquals(HDDSVersion.SOFTWARE_VERSION, versionManager.getVersionForClient());
+ }
+ }
+
+ @Test
+ public void testGetVersionForClientWhenPreZdu() throws Exception {
+ // A pre-ZDU apparent version clamps to the last pre-ZDU wire version.
+ try (ScmVersionManager versionManager = createManager(INITIAL_VERSION.serialize())) {
- assertEquals(HDDSVersion.STREAM_BLOCK_SUPPORT, versionManager.getVersionForClient());
++ assertEquals(HDDSVersion.SHORT_CIRCUIT_READS, versionManager.getVersionForClient());
+ }
+ }
+
+ private static void assertScmApparentVersionOnDisk(OzoneConfiguration conf, int expected)
+ throws IOException {
+ SCMStorageConfig reloaded = new SCMStorageConfig(conf);
+ assertEquals(expected, reloaded.getApparentVersion());
+ }
+}
diff --cc hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java
index fd1650683b0,3a2e396fd5f..74a68a96b8c
--- a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java
+++ b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java
@@@ -31,7 -31,7 +31,8 @@@ import org.apache.hadoop.hdds.client.Bl
import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.ClientVersion;
+ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.debug.OzoneDebug;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index bf84d7da73f,893fc90138c..3b898c8dd83
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@@ -1858,11 -1865,24 +1859,24 @@@ public class RpcClient implements Clien
@Override
public OzoneKeyDetails getS3KeyDetails(String bucketName, String keyName,
int partNumber) throws IOException {
+ return getOzoneKeyDetails(
+ getS3PartOmKeyInfo(bucketName, keyName, partNumber, false));
+ }
+
+ @Override
+ public OzoneKey headS3Object(String bucketName, String keyName,
+ int partNumber) throws IOException {
+ return OzoneKey.fromKeyInfo(
+ getS3PartOmKeyInfo(bucketName, keyName, partNumber, true));
+ }
+
+ private OmKeyInfo getS3PartOmKeyInfo(String bucketName, String keyName,
+ int partNumber, boolean isHeadOp) throws IOException {
OmKeyInfo keyInfo;
- if (omVersion.compareTo(OzoneManagerVersion.S3_PART_AWARE_GET) >= 0) {
+ if (OzoneManagerVersion.S3_PART_AWARE_GET.isSupportedBy(omVersion)) {
- keyInfo = getS3PartKeyInfo(bucketName, keyName, partNumber);
+ keyInfo = getS3PartKeyInfo(bucketName, keyName, partNumber, isHeadOp);
} else {
- keyInfo = getS3KeyInfo(bucketName, keyName, false);
+ keyInfo = getS3KeyInfo(bucketName, keyName, isHeadOp);
List<OmKeyLocationInfo> filteredKeyLocationInfo = keyInfo
.getLatestVersionLocations().getBlocksLatestVersionOnly().stream()
.filter(omKeyLocationInfo -> omKeyLocationInfo.getPartNumber() ==
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 842d6ef353c,9e9622c6411..6de380da870
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@@ -260,13 -261,11 +261,15 @@@ public final class OmUtils
case GetSnapshotInfo:
case GetObjectTagging:
case GetBucketTagging:
- return true;
case GetQuotaRepairStatus:
case StartQuotaRepair:
+ // Prepare and CancelPrepare are now no-ops, but still produce responses for compatability.
+ // They are marked as read-only since they don't modify state anymore.
+ case Prepare:
+ case CancelPrepare:
+ case QueryUpgradeStatus:
+ case GetLifecycleConfiguration:
+ case GetLifecycleServiceStatus:
return true;
case CreateVolume:
case SetVolumeProperty:
@@@ -465,8 -470,11 +474,12 @@@
case GetQuotaRepairStatus:
// Quota repair lifecycle request should be initiated by the leader
case DBUpdates: // We are currently only interested on the leader DB info
+ case SetLifecycleConfiguration:
+ case DeleteLifecycleConfiguration:
+ case SetLifecycleServiceStatus:
+ case SaveLifecycleScanState:
case UnknownCommand:
+ case QueryUpgradeStatus:
return false;
case EchoRPC:
return omRequest.getEchoRPCRequest().getReadOnly();
diff --cc hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java
index 0fd5234ba91,8ea2d8c2125..d47a34f7a50
--- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java
+++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java
@@@ -49,8 -49,8 +49,9 @@@ import org.apache.hadoop.hdds.scm.conta
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/TestDecommissionAndMaintenance.java
index 7ccc131f53f,754fd4a3297..b17a0a99952
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/TestDecommissionAndMaintenance.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/TestDecommissionAndMaintenance.java
@@@ -148,7 -150,10 +149,9 @@@ public class TestDecommissionAndMainten
conf.setFromObject(replicationConf);
MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(DATANODE_COUNT);
+ .setNumDatanodes(DATANODE_COUNT)
+ .setDatanodeFactory(UniformDatanodesFactory.newBuilder()
- .setLayoutVersion(maxLayoutVersion())
+ .build());
clusterProvider = new MiniOzoneClusterProvider(builder, 9);
}
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
index 38738fcc8e7,7caf91294d7..48d159de691
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
@@@ -154,12 -165,21 +154,13 @@@ public class TestDNDataDistributionFina
// Validate pre-finalization state
validatePreDataDistributionFeatureState();
- // Now trigger finalization
- Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(
- () -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- LOG.info("finalization client failed. This may be expected if the" +
- " test injected failures.", ex);
- }
- });
-
// Wait for finalization to complete
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
+ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
+
+ // Verify finalization completed
+ assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization());
+ assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION));
// Create more data and deletions to test post-finalization behavior
String keyName3 = "testKey3";
@@@ -190,10 -210,19 +191,10 @@@
out.write(data);
}
bucket.deleteKey(keyName);
- Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(
- () -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- LOG.info("finalization client failed. This may be expected if the" +
- " test injected failures.", ex);
- }
- });
- // Wait for finalization
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
+ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
-
+ assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization());
+ assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION));
// Verify the system can handle scenarios where pendingDeleteBlockCount
// might be missing and needs recalculation
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
index b3264137626,a1a8ca88c3d..d92dd4030fb
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java
@@@ -75,6 -81,8 +76,7 @@@ import org.apache.hadoop.ozone.client.O
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.common.DeletedBlock;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
+ import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalizationExecutor;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.tag.Flaky;
import org.junit.jupiter.api.AfterEach;
@@@ -160,18 -193,19 +162,19 @@@ public class TestScmDataDistributionFin
@Test
@Flaky("HDDS-14050")
public void testFinalizationEmptyClusterDataDistribution() throws Exception {
- init(new OzoneConfiguration(), null, true);
+ init(new OzoneConfiguration());
assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
+ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Make sure old leader has caught up and all SCMs have finalized.
waitForScmsToFinalize(cluster.getStorageContainerManagersList());
+ assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION));
HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
+ cluster.getStorageContainerManagersList(), 0);
HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
for (StorageContainerManager scm: cluster.getStorageContainerManagersList()) {
@@@ -266,15 -300,25 +269,16 @@@
flushDBTransactionBuffer(activeSCM);
assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
- finalizationFuture = Executors.newSingleThreadExecutor().submit(
- () -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- LOG.info("finalization client failed. This may be expected if the" +
- " test injected failures.", ex);
- }
- });
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
+ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
// Make sure old leader has caught up and all SCMs have finalized.
waitForScmsToFinalize(cluster.getStorageContainerManagersList());
+ assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION));
HddsUpgradeTestUtils.testPostUpgradeConditionsSCM(
- cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
+ cluster.getStorageContainerManagersList(), 0);
HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes(
- cluster.getHddsDatanodes(), 0, CLOSED);
+ cluster.getHddsDatanodes(), 0);
assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary());
for (StorageContainerManager scm: cluster.getStorageContainerManagersList()) {
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
index c915c393014,c011c774c77..278fda0f5ee
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
@@@ -61,9 -61,8 +61,9 @@@ import org.apache.hadoop.ozone.client.i
import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
- import org.apache.hadoop.ozone.container.TestHelper;
+ import org.apache.hadoop.ozone.container.OzoneTestHelper;
import org.apache.ozone.test.tag.Flaky;
+import org.apache.ozone.test.tag.Unhealthy;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestInstance;
@@@ -334,11 -350,11 +352,12 @@@ public class TestBlockDataStreamOutput
}
}
+ @Unhealthy("Requires HDDS-16044 to finish implementing datanode version passing to client.")
@ParameterizedTest
@MethodSource("clientParameters")
- public void testDatanodeVersion(boolean flushDelay) throws Exception {
- OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay);
+ public void testDatanodeVersion(boolean flushDelay, boolean putBlockOnCloseEnabled)
+ throws Exception {
+ OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay, putBlockOnCloseEnabled);
try (OzoneClient client = newClient(cluster.getConf(), config)) {
// Verify all DNs internally have versions set correctly
List<HddsDatanodeService> dns = cluster.getHddsDatanodes();
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestFinalizeBlock.java
index 6eb50bdfc45,6eb50bdfc45..37089f72fa5
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestFinalizeBlock.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestFinalizeBlock.java
@@@ -108,7 -108,7 +108,8 @@@ public class TestFinalizeBlock
conf.setFromObject(scmConfig);
cluster = MiniOzoneCluster.newBuilder(conf)
-- .setNumDatanodes(1).build();
++ .setNumDatanodes(1)
++ .build();
cluster.waitForClusterToBeReady();
cluster.waitForPipelineTobeReady(ONE, 30000);
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
index 086416f85d5,be0f9909977..16228cfbc02
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
@@@ -30,203 -30,45 +30,203 @@@ import static org.junit.jupiter.api.Ass
import java.io.IOException;
import java.util.List;
import java.util.UUID;
-import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
++import org.apache.hadoop.ozone.DataTestUtil;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
-import org.apache.hadoop.ozone.audit.AuditEventStatus;
-import org.apache.hadoop.ozone.audit.AuditLogTestUtils;
-import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.OzoneManagerVersion;
- import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.conf.OMClientConfig;
+import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages;
-import org.apache.ratis.util.LifeCycle;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeEach;
+import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
+import org.apache.hadoop.ozone.upgrade.RatisBasedVersionManager;
+import org.apache.ozone.test.GenericTestUtils.LogCapturer;
import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tests for OM upgrade finalization.
- * TODO: can be merged into class with other OM tests with per-method cluster
*/
class TestOMUpgradeFinalization {
- static {
- AuditLogTestUtils.enableAuditLog();
- }
- @BeforeEach
- public void setup() throws Exception {
- AuditLogTestUtils.truncateAuditLogFile();
+ private static final Logger LOG = LoggerFactory.getLogger(TestOMUpgradeFinalization.class);
+
+ // Force Ratis to take a snapshot and purge logs after only a few
+ // transactions, so a lagging OM has to install a snapshot on startup.
+ private static final long SNAPSHOT_THRESHOLD = 5;
+ private static final int LOG_PURGE_GAP = 5;
+
+ /**
+ * A lagging OM that has to install a Ratis snapshot should finalize
+ * from that snapshot.
+ * When finalization runs on the active OMs while one OM is down, the down OM
+ * cannot replay the finalization from the (purged) logs, so it must pick up
+ * the finalized version from the checkpoint it downloads from the leader.
+ */
+ @Test
+ void testFinalizationFromSnapshot() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms");
+ // Force frequent snapshots and log purge so the inactive OM must install a
+ // snapshot from the leader (rather than replay logs) when it starts.
+ conf.setInt(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP, LOG_PURGE_GAP);
+ conf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_KEY, 16, StorageUnit.KB);
+ conf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY, 16, StorageUnit.KB);
+ conf.setLong(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY, SNAPSHOT_THRESHOLD);
+
+ // Use a short client RPC timeout so the first request fails over quickly to
+ // a responsive OM instead of blocking on the 15-minute default if it lands on the inactive one.
+ OMClientConfig clientConfig = conf.getObject(OMClientConfig.class);
+ clientConfig.setRpcTimeOut(TimeUnit.SECONDS.toMillis(5));
+ conf.setFromObject(clientConfig);
+
+ LogCapturer versionManagerLogCapture = LogCapturer.captureLogs(RatisBasedVersionManager.class);
+ LogCapturer omLogCapture = LogCapturer.captureLogs(OzoneManager.class);
+
+ try (MiniOzoneHAClusterImpl cluster = newCluster(conf)) {
+ LOG.info("Waiting for cluster to be ready");
+ cluster.waitForClusterToBeReady();
+ OzoneManager followerOM = cluster.getInactiveOM().next();
+ LOG.info("Cluster ready with inactive OM: {}", followerOM);
+
+ try (OzoneClient client = cluster.newClient()) {
+ LOG.info("Client created");
+ ObjectStore objectStore = client.getObjectStore();
+ LOG.info("Object store created");
+
+ // The active OMs start un-finalized: the apparent version is the initial
+ // version and no version has been written to the DB yet.
+ for (OzoneManager om : cluster.getOzoneManagersList()) {
+ if (cluster.isOMActive(om.getOMNodeId())) {
+ assertEquals(INITIAL_VERSION, om.getVersionManager().getApparentVersion());
+ assertNull(om.getMetadataManager().getMetaTable().get(APPARENT_VERSION_KEY));
+ }
+ }
+
+ // Finalize the running (active) OMs.
+ LOG.info("Finalizing OMs");
+ OzoneManagerProtocol omClient = objectStore.getClientProxy().getOzoneManagerClient();
+ omClient.forceFinalizeUpgrade();
+ OMUpgradeTestUtils.waitForFinalization(omClient);
+ LOG.info("Finalized active OMs");
+
+ // Write enough keys to advance the log index well past the purge gap so
+ // the leader's logs covering finalization are purged. This forces the
+ // follower to install a snapshot instead of replaying the logs.
+ OzoneManager leaderOM = cluster.getOMLeader();
+ assertNotNull(leaderOM);
+ OzoneManagerRatisServer leaderRatisServer = leaderOM.getOmRatisServer();
- OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster.newClient());
++ OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(cluster.newClient());
+ long targetLogIndex = leaderRatisServer.getLastAppliedTermIndex().getIndex() + 100;
+ LOG.info("Writing keys to advance log index");
+ writeKeysToIncreaseLogIndex(leaderRatisServer, targetLogIndex, bucket);
+
+ // Start the inactive OM. It downloads the leader's checkpoint (which
+ // already contains the finalized apparent version) and finalizes from it.
+ LOG.info("Starting inactive OM");
+ cluster.startInactiveOM(followerOM.getOMNodeId());
+
+ // Wait for the follower to finish installing the snapshot and catch up
+ // to the leader before reading its DB and shutting down, so shutdown
+ // does not race with the tail of the snapshot install.
+ LOG.info("Inactive OM started, waiting to catch up with leader");
+ leaderOM = cluster.getOMLeader();
+ assertNotNull(leaderOM);
+ long leaderIndex = leaderOM.getOmRatisServer().getLastAppliedTermIndex().getIndex();
+ waitFor(() -> {
+ boolean running = followerOM.isRunning();
+ // Make sure the snapshot install finishes completely so it does not block cluster shutdown on test cleanup.
+ boolean snapshotFinished = omLogCapture.getOutput().contains("Install Checkpoint is finished");
+ long followerIndex = followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex();
+ LOG.info("Waiting for follower to catch up with leader. Running? {} Snapshot finished? {}" +
+ " Follower index = {} leader index = {}",
+ running, snapshotFinished, followerIndex, leaderIndex);
+ return running && snapshotFinished && followerIndex >= leaderIndex;
+ }, 1000, 60000);
+ LOG.info("Inactive OM has caught up with leader, checking finalization state");
+ omLogCapture.stopCapturing();
+
+ // The follower finalizes to the software version, picked up from the installed snapshot.
+ assertEquals(OzoneManagerVersion.SOFTWARE_VERSION, followerOM.getVersionManager().getApparentVersion());
+
+ String dbVersion = followerOM.getMetadataManager().getMetaTable().get(APPARENT_VERSION_KEY);
+ assertNotNull(dbVersion);
+ assertEquals(OzoneManagerVersion.SOFTWARE_VERSION.serialize(), Integer.parseInt(dbVersion));
+
+ // Confirm finalization happened via snapshot install, not log replay.
+ assertThat(versionManagerLogCapture.getOutput()).contains("New snapshot received with higher apparent version");
+ versionManagerLogCapture.stopCapturing();
+ }
+ }
}
- @AfterAll
- public static void shutdown() {
- AuditLogTestUtils.deleteAuditLogFile();
+ /**
+ * OM's reported finalization status should move UNFINALIZED -> IN_PROGRESS (once the in-progress
+ * marker is present) -> FINALIZED.
+ */
+ @Test
+ void testOmFinalizationStatusTransitions() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms");
+ conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, INITIAL_VERSION.serialize());
+
+ try (MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(1)
+ .build()) {
+ cluster.waitForClusterToBeReady();
+ OzoneManager om = cluster.getOzoneManager();
+
+ try (OzoneClient client = cluster.newClient()) {
+ OzoneManagerProtocol omClient = client.getObjectStore().getClientProxy().getOzoneManagerClient();
+
+ // Before finalization: no in-progress marker, OM reports UNFINALIZED.
+ assertNull(om.getMetadataManager().getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY));
+ assertEquals(HddsProtos.FinalizationStatus.UNFINALIZED,
+ omClient.queryUpgradeStatus().getOmFinalizationStatus());
+
+ // With the in-progress marker present but the OM not yet finalized, OM reports IN_PROGRESS.
+ om.getMetadataManager().getMetaTable().put(FINALIZATION_IN_PROGRESS_KEY, "ignored");
+ assertEquals(HddsProtos.FinalizationStatus.IN_PROGRESS,
+ omClient.queryUpgradeStatus().getOmFinalizationStatus());
+ om.getMetadataManager().getMetaTable().delete(FINALIZATION_IN_PROGRESS_KEY);
+
+ // After finalization completes, OM reports FINALIZED.
+ omClient.finalizeUpgrade();
+ OMUpgradeTestUtils.waitForFinalization(omClient);
+ assertEquals(HddsProtos.FinalizationStatus.FINALIZED,
+ omClient.queryUpgradeStatus().getOmFinalizationStatus());
+ }
+ }
}
+ /**
+ * The OM version advertised in its {@link ServiceInfo} should track
+ * {@code getVersionForClient()} across finalization: the last pre-ZDU client
+ * version before ZDU is finalized, and the software version after.
+ *
+ * On an HA cluster every OM should report the same client-facing version, both
+ * for itself and for its peers, in every OM's {@code getServiceList()}.
+ */
@Test
- void testOMUpgradeFinalizationWithOneOMDown() throws Exception {
+ void testServiceInfoOmVersionTracksClientVersion() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- try (MiniOzoneHAClusterImpl cluster = newCluster(conf)) {
+ conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms");
+ conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, INITIAL_VERSION.serialize());
+
+ MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf);
+ builder.setOMServiceId(UUID.randomUUID().toString())
+ .setNumOfOzoneManagers(3)
+ .setNumOfActiveOMs(3)
+ .withoutDatanodes();
+ try (MiniOzoneHAClusterImpl cluster = builder.build()) {
cluster.waitForClusterToBeReady();
try (OzoneClient client = cluster.newClient()) {
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupInvalidApparentVersion.java
index 65e86fbd03d,78ec2713334..d0aafe818d3
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupInvalidApparentVersion.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupInvalidApparentVersion.java
@@@ -46,19 -46,9 +46,21 @@@ public class TestOmStartupInvalidAppare
private Path folder;
@Test
- public void testStartupSlvLessThanMlv() throws Exception {
- // Add subdirectories under the temporary folder where the version file
- // will be placed.
+ public void testStartupFailsWhenApparentVersionBetweenLastLayoutFeatureAndZdu()
+ throws Exception {
- assertStartupFailsWithComponentVersionMessage(OMLayoutFeature.SNAPSHOT_DEFRAG.serialize() + 1);
++ OMLayoutFeature[] features = OMLayoutFeature.values();
++ int lastLayoutVersion = features[features.length - 1].serialize();
++ assertStartupFailsWithComponentVersionMessage(lastLayoutVersion + 1);
+ }
+
+ @Test
+ public void testStartupFailsWhenApparentVersionBeyondLastKnownComponentVersion()
+ throws Exception {
+ assertStartupFailsWithComponentVersionMessage(SOFTWARE_VERSION.serialize() + 1);
+ }
+
+ private void assertStartupFailsWithComponentVersionMessage(int serializedApparentVersion)
+ throws Exception {
String subDir = folder.toAbsolutePath() + "/om/current";
File omSubdir = Files.createDirectories(Paths.get(subDir)).toFile();
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
index a0fd40b5163,d3364c1f742..8f7a06bb146
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java
@@@ -20,6 -20,8 +20,7 @@@ package org.apache.hadoop.ozone.om.serv
import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.HBASE_SUPPORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.params.provider.Arguments.arguments;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.timeout;
@@@ -41,8 -45,11 +42,9 @@@ import org.apache.hadoop.hdds.conf.Ozon
import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMPerformanceMetrics;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext;
+ import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.UniformDatanodesFactory;
@@@ -51,7 -58,9 +53,8 @@@ import org.apache.hadoop.ozone.client.O
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.common.DeletedBlock;
+ import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.hadoop.ozone.om.helpers.QuotaUtil;
-import org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@@@ -135,9 -152,18 +138,10 @@@ public class TestBlockDeletionService
GenericTestUtils.waitFor(() -> metrics.getDeleteKeySuccessBlocks() - initialSuccessBlocks == 1, 50, 1000);
GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000);
-- // UPGRADE SCM (if specified)
+ // Step 5: wait for finalizing upgrade
- Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(() -> {
- try {
- scmClient.finalizeScmUpgrade(CLIENT_ID);
- } catch (IOException ex) {
- fail("finalization client failed", ex);
- }
- });
- finalizationFuture.get();
- HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+ scmClient.finalizeUpgrade();
+ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient);
+ assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION));
// POST-UPGRADE
//Step 6: Repeat the same steps in pre-upgrade
diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index 340f9d160e5,00b070e29d2..8561b8e4aa2
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@@ -100,7 -102,7 +102,8 @@@ import org.apache.hadoop.ozone.om.servi
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ToolRunner;
import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.tag.Unhealthy;
+ import org.apache.ratis.server.RaftServerConfigKeys;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index f243e8e9d0a,2a27af60754..1b6ada0edca
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -161,8 -161,12 +161,14 @@@ enum Type
PutBucketTagging = 144;
GetBucketTagging = 145;
DeleteBucketTagging = 146;
- StartFinalizeUpgrade = 147;
- QueryUpgradeStatus = 148;
+ SetLifecycleConfiguration = 147;
+ GetLifecycleConfiguration = 148;
+ DeleteLifecycleConfiguration = 149;
+ GetLifecycleServiceStatus = 150;
+ SetLifecycleServiceStatus = 151;
+ SaveLifecycleScanState = 152;
++ StartFinalizeUpgrade = 153;
++ QueryUpgradeStatus = 154;
}
enum SafeMode {
@@@ -321,8 -324,12 +326,14 @@@ message OMRequest
optional GetBucketTaggingRequest getBucketTaggingRequest = 146;
// TODO: DeleteBucketTagging — clears tags on target bucket (link resolves in OM).
optional DeleteBucketTaggingRequest deleteBucketTaggingRequest = 147;
- optional StartFinalizeUpgradeRequest startFinalizeUpgradeRequest = 148;
- optional QueryUpgradeStatusRequest queryUpgradeStatusRequest = 149;
+ optional SetLifecycleConfigurationRequest setLifecycleConfigurationRequest = 148;
+ optional GetLifecycleConfigurationRequest getLifecycleConfigurationRequest = 149;
+ optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 150;
+ optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 151;
+ optional SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest = 152;
+ optional SaveLifecycleScanStateRequest saveLifecycleScanStateRequest = 153;
++ optional StartFinalizeUpgradeRequest startFinalizeUpgradeRequest = 154;
++ optional QueryUpgradeStatusRequest queryUpgradeStatusRequest = 155;
}
message OMResponse {
@@@ -456,16 -463,21 +467,21 @@@
optional GetObjectTaggingResponse getObjectTaggingResponse = 140;
optional PutObjectTaggingResponse putObjectTaggingResponse = 141;
optional DeleteObjectTaggingResponse deleteObjectTaggingResponse = 142;
-
optional SubmitSnapshotDiffResponse submitSnapshotDiffResponse = 143;
-
// TODO: Empty ack after OM applies tag set to OmBucketInfo.
optional PutBucketTaggingResponse putBucketTaggingResponse = 144;
// TODO: Tag list for S3G GetBucketTagging XML; empty if no tags.
optional GetBucketTaggingResponse getBucketTaggingResponse = 145;
// TODO: Empty ack after OM clears BucketInfo.tags.
optional DeleteBucketTaggingResponse deleteBucketTaggingResponse = 146;
- optional StartFinalizeUpgradeResponse startFinalizeUpgradeResponse = 147;
- optional QueryUpgradeStatusResponse queryUpgradeStatusResponse = 148;
-
+ optional SetLifecycleConfigurationResponse setLifecycleConfigurationResponse = 147;
+ optional GetLifecycleConfigurationResponse getLifecycleConfigurationResponse = 148;
+ optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 149;
+ optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 150;
+ optional SetLifecycleServiceStatusResponse setLifecycleServiceStatusResponse = 151;
+ optional SaveLifecycleScanStateResponse saveLifecycleScanStateResponse = 152;
++ optional StartFinalizeUpgradeResponse startFinalizeUpgradeResponse = 153;
++ optional QueryUpgradeStatusResponse queryUpgradeStatusResponse = 154;
}
enum Status {
diff --cc hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java
index d1e90d8c346,b4a22c67dff..85f8825372e
--- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java
+++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java
@@@ -103,14 -105,17 +103,20 @@@ public class UniformDatanodesFactory im
Files.createDirectories(ratisDir);
dnConf.set(HDDS_CONTAINER_RATIS_DATANODE_STORAGE_DIR, ratisDir.toString());
- if (layoutVersion != null) {
- DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(
- dnConf, UUID.randomUUID().toString(), layoutVersion);
+ if (apparentVersion != null) {
+ DatanodeStorage layoutStorage = new DatanodeStorage(
+ dnConf, UUID.randomUUID().toString(), apparentVersion.serialize());
+ layoutStorage.initialize();
++ } else {
++ // TODO HDDS-16206 workaround. Once that is resolved MiniOzoneCluster Datanodes should start with
++ // SOFTWARE_VERSION by default.
++ DatanodeStorage layoutStorage = new DatanodeStorage(
++ dnConf, UUID.randomUUID().toString(), HDDSVersion.SOFTWARE_VERSION.serialize());
+ layoutStorage.initialize();
}
- if (initialVersion != null) {
- dnConf.setInt(TESTING_DATANODE_VERSION_INITIAL, initialVersion.toProtoValue());
- }
if (currentVersion != null) {
- dnConf.setInt(TESTING_DATANODE_VERSION_CURRENT, currentVersion.toProtoValue());
+ dnConf.setInt(TESTING_DATANODE_VERSION_CURRENT, currentVersion.serialize());
}
dnConf.set(HDDS_RATIS_LEADER_FIRST_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY, "1s");
dnConf.set(HDDS_INITIAL_HEARTBEAT_INTERVAL, "500ms");
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a8ce1e53508,cd850e9e6a5..79c8f0314be
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -312,9 -317,9 +319,10 @@@ import org.apache.hadoop.ozone.protocol
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.EchoRPCResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExtendedUserAccessIdInfo;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.QueryUpgradeStatusResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantState;
@@@ -486,7 -492,11 +494,9 @@@ public final class OzoneManager extend
private final boolean isS3MultiTenancyEnabled;
private final boolean isStrictS3;
private ExitManager exitManager;
+ /** Test-only hook to fail a checkpoint-install DB backup part way through. */
+ private FaultInjector checkpointBackupInjector;
- private OzoneManagerPrepareState prepareState;
-
private boolean isBootstrapping = false;
private boolean isForcedBootstrapping = false;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
index 00000000000,c36097fd4d3..77d27d6c313
mode 000000,100644..100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
@@@ -1,0 -1,2172 +1,2172 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.ozone.om.service;
+
+ import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
+ import static org.apache.hadoop.fs.ozone.OzoneTrashPolicy.CURRENT;
+ import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE_DEFAULT;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK_DEFAULT;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED;
+ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT;
+ import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
+
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.base.Preconditions;
+ import com.google.protobuf.ServiceException;
+ import jakarta.annotation.Nullable;
+ import java.io.IOException;
+ import java.nio.file.Paths;
+ import java.security.PrivilegedExceptionAction;
+ import java.time.Instant;
+ import java.util.ArrayDeque;
+ import java.util.ArrayList;
+ import java.util.Collections;
+ import java.util.Comparator;
+ import java.util.Deque;
+ import java.util.HashSet;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+ import java.util.concurrent.ConcurrentHashMap;
+ import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.concurrent.atomic.AtomicLong;
+ import java.util.stream.Collectors;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.hdds.conf.ConfigurationSource;
+ import org.apache.hadoop.hdds.conf.StorageUnit;
+ import org.apache.hadoop.hdds.utils.BackgroundService;
+ import org.apache.hadoop.hdds.utils.BackgroundTask;
+ import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+ import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+ import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+ import org.apache.hadoop.hdds.utils.FaultInjector;
+ import org.apache.hadoop.hdds.utils.db.Table;
+ import org.apache.hadoop.hdds.utils.db.TableIterator;
+ import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+ import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+ import org.apache.hadoop.ozone.ClientVersion;
+ import org.apache.hadoop.ozone.OzoneConsts;
+ import org.apache.hadoop.ozone.om.KeyManager;
+ import org.apache.hadoop.ozone.om.OMConfigKeys;
+ import org.apache.hadoop.ozone.om.OMMetadataManager;
+ import org.apache.hadoop.ozone.om.OzoneManager;
+ import org.apache.hadoop.ozone.om.OzoneTrash;
+ import org.apache.hadoop.ozone.om.exceptions.OMException;
+ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmLCFilter;
+ import org.apache.hadoop.ozone.om.helpers.OmLCRule;
+ import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
+ import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
+ import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
+ import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload;
+ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+ import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
+ import org.apache.hadoop.ozone.om.request.OMClientRequest;
+ import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateDirectoryRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyError;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RequestSource;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest;
+ import org.apache.hadoop.security.UserGroupInformation;
+ import org.apache.hadoop.util.Time;
+ import org.apache.ratis.protocol.ClientId;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * This is the background service to manage object lifecycle based on bucket lifecycle configuration.
+ */
+ public class KeyLifecycleService extends BackgroundService {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(KeyLifecycleService.class);
+
+ private final OzoneManager ozoneManager;
+ private int keyDeleteBatchSize;
+ private int listMaxSize;
+ private int mpuAbortLimitPerTask;
+ private long cachedDirMaxCount;
+ private final AtomicBoolean suspended;
+ private final AtomicBoolean isServiceEnabled;
+ private final AtomicBoolean moveToTrashEnabled;
+ private KeyLifecycleServiceMetrics metrics;
+ // A set of bucket name that have LifecycleActionTask scheduled
+ private final ConcurrentHashMap<String, LifecycleActionTask> inFlight;
+ private OMMetadataManager omMetadataManager;
+ private int ratisByteLimit;
+ private long stateSaveIntervalMs;
+ private long maxKeysProcessedPerState;
+ private ClientId clientId = ClientId.randomId();
+ private AtomicLong callId = new AtomicLong(0);
+ private OzoneTrash ozoneTrash;
+ private static List<FaultInjector> injectors;
+ private static boolean test = false;
+ private static List<RuleListWithDirectoryList> consolidatedRuleList;
+
+ public KeyLifecycleService(OzoneManager ozoneManager,
+ KeyManager manager, long serviceInterval,
+ long serviceTimeout, int poolSize,
+ ConfigurationSource conf) {
+ super(KeyLifecycleService.class.getSimpleName(), serviceInterval, TimeUnit.MILLISECONDS,
+ poolSize, serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.keyDeleteBatchSize = conf.getInt(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE,
+ OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE_DEFAULT);
+ Preconditions.checkArgument(keyDeleteBatchSize > 0,
+ OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE + " should be a positive value.");
+ this.listMaxSize = keyDeleteBatchSize >= 10000 ? keyDeleteBatchSize : 10000;
+ this.mpuAbortLimitPerTask = conf.getInt(OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK,
+ OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK_DEFAULT);
+ Preconditions.checkArgument(mpuAbortLimitPerTask > 0,
+ OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK + " should be a positive value.");
+ this.cachedDirMaxCount = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT,
+ OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT);
+ this.suspended = new AtomicBoolean(false);
+ this.metrics = KeyLifecycleServiceMetrics.create();
+ this.isServiceEnabled = new AtomicBoolean(conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED,
+ OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT));
+ this.moveToTrashEnabled = new AtomicBoolean(conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED,
+ OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT));
+ this.stateSaveIntervalMs = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT);
+ if (!test && stateSaveIntervalMs <= 0) {
+ LOG.warn("Illegal value {} for Property {}. Set {} to {}", stateSaveIntervalMs,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT);
+ stateSaveIntervalMs = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT;
+ }
+ this.maxKeysProcessedPerState = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT);
+ if (!test && maxKeysProcessedPerState <= 0) {
+ LOG.warn("Illegal value {} for Property {}. Set {} to {}", maxKeysProcessedPerState,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT);
+ maxKeysProcessedPerState = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT;
+ }
+ LOG.info("stateSaveIntervalMs = {}, maxKeysProcessedPerState = {}", stateSaveIntervalMs, maxKeysProcessedPerState);
+ this.inFlight = new ConcurrentHashMap();
+ this.omMetadataManager = ozoneManager.getMetadataManager();
+ int limit = (int) conf.getStorageSize(
+ OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT,
+ OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT,
+ StorageUnit.BYTES);
+ // always go to 90% of max limit for request as other header will be added
+ this.ratisByteLimit = (int) (limit * 0.9);
+ }
+
+ @Override
+ public BackgroundTaskQueue getTasks() {
+ BackgroundTaskQueue queue = new BackgroundTaskQueue();
+ if (!shouldRun()) {
+ return queue;
+ }
+
+ List<OmLifecycleConfiguration> lifecycleConfigurationList = null;
+ try {
+ lifecycleConfigurationList = omMetadataManager.listLifecycleConfigurations();
+ } catch (OMException e) {
+ LOG.error("Failed to list lifecycle configurations", e);
+ return queue;
+ }
+ for (OmLifecycleConfiguration lifecycleConfiguration : lifecycleConfigurationList) {
+ try {
+ lifecycleConfiguration.valid();
+ } catch (OMException e) {
+ LOG.error("Skip invalid lifecycle configuration for {}/{}: LifecycleConfiguration:\n {}",
+ lifecycleConfiguration.getVolume(), lifecycleConfiguration.getBucket(),
+ lifecycleConfiguration.getProtobuf(), e);
+ continue;
+ }
+ String bucketKey = omMetadataManager.getBucketKey(lifecycleConfiguration.getVolume(),
+ lifecycleConfiguration.getBucket());
+ if (lifecycleConfiguration.getRules().stream().anyMatch(r -> r.isEnabled())) {
+ LifecycleActionTask task = new LifecycleActionTask(lifecycleConfiguration);
+ if (this.inFlight.putIfAbsent(bucketKey, task) == null) {
+ queue.add(task);
+ LOG.info("LifecycleActionTask of {} is scheduled", bucketKey);
+ } else {
+ metrics.incrNumSkippedTask();
+ LOG.info("LifecycleActionTask of {} is already running", bucketKey);
+ }
+ } else {
+ LOG.info("LifecycleConfiguration of {} is not enabled", bucketKey);
+ }
+ }
+ LOG.info("{} LifecycleActionTasks scheduled", queue.size());
+ return queue;
+ }
+
+ private boolean shouldRun() {
+ if (getOzoneManager() == null) {
+ // OzoneManager can be null for testing
+ return true;
+ }
+ return isServiceEnabled.get() && !suspended.get() && getOzoneManager().isLeaderReady();
+ }
+
+ public KeyLifecycleServiceMetrics getMetrics() {
+ return metrics;
+ }
+
+ public OzoneManager getOzoneManager() {
+ return ozoneManager;
+ }
+
+ /**
+ * Suspend the service.
+ */
+ public void suspend() {
+ suspended.set(true);
+ }
+
+ /**
+ * Resume the service if suspended.
+ */
+ public void resume() {
+ suspended.set(false);
+ }
+
+ public boolean isSuspended() {
+ return suspended.get();
+ }
+
+ @Override
+ public void shutdown() {
+ super.shutdown();
+ KeyLifecycleServiceMetrics.unregister();
+ }
+
+ /**
+ * Build a GetLifecycleServiceStatusResponse instance.
+ * @return GetLifecycleServiceStatusResponse instance
+ */
+ public GetLifecycleServiceStatusResponse status() {
+ Set<String> runningBuckets = new HashSet<>(inFlight.keySet());
+ return GetLifecycleServiceStatusResponse.newBuilder()
+ .setIsEnabled(isServiceEnabled.get())
+ .setIsSuspended(suspended.get())
+ .addAllRunningBuckets(runningBuckets)
+ .build();
+ }
+
+ /**
+ * A lifecycle action task for one specific bucket, scanning OM DB and evaluating if any existing
+ * object/key qualified for expiration according to bucket's lifecycle configuration, and sending
+ * key delete command respectively.
+ */
+ public final class LifecycleActionTask implements BackgroundTask {
+ private final OmLifecycleConfiguration policy;
+ private long taskStartTime;
+ private long numKeyIterated = 0;
+ private long numDirIterated = 0;
+ private long numDirDeleted = 0;
+ private long numKeyDeleted = 0;
+ private long sizeKeyDeleted = 0;
+ private long numKeyRenamed = 0;
+ private long sizeKeyRenamed = 0;
+ private long numDirRenamed = 0;
+ private long numMultipartUploadIterated = 0;
+ private long numMultipartUploadAborted = 0;
+ private String lastScannedKey;
+ private String lastScannedDir;
+ private String lastScannedDirKey;
+
+ private long lastStateSaveTime = Time.monotonicNow();
+ private long lastStateSaveKeyCount = 0;
+
+ private boolean shouldSaveState() {
+ if ((Time.monotonicNow() - lastStateSaveTime) > stateSaveIntervalMs ||
+ (numKeyIterated - lastStateSaveKeyCount) >= maxKeysProcessedPerState) {
+ return true;
+ }
+ return false;
+ }
+
+ public LifecycleActionTask(OmLifecycleConfiguration lcConfig) {
+ this.policy = lcConfig;
+ }
+
+ @Override
+ public int getPriority() {
+ return 0;
+ }
+
+ @Override
+ public BackgroundTaskResult call() {
+ EmptyTaskResult result = EmptyTaskResult.newResult();
+ String bucketKey = omMetadataManager.getBucketKey(policy.getVolume(), policy.getBucket());
+ // Check if this is the Leader OM. If not leader, no need to execute this task.
+ if (shouldRun()) {
+ LOG.info("Running LifecycleActionTask {}", bucketKey);
+ taskStartTime = Time.monotonicNow();
+ lastStateSaveTime = taskStartTime;
+ OmBucketInfo bucket;
+ try {
+ if (getInjector(0) != null) {
+ getInjector(0).pause();
+ }
+ bucket = omMetadataManager.getBucketTable().get(bucketKey);
+ if (bucket == null) {
+ LOG.warn("Bucket {} cannot be found, might be deleted during this task's execution", bucketKey);
+ onFailure(bucketKey);
+ return result;
+ }
+ if (bucket.getObjectID() != policy.getBucketObjectID()) {
+ LOG.warn("Bucket object ID doesn't match. ID in bucket is {}, ID in LifecycleConfiguration is {}.",
+ bucket.getObjectID(), policy.getBucketObjectID());
+ onFailure(bucketKey);
+ return result;
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to get Bucket {}", bucketKey, e);
+ onFailure(bucketKey);
+ return result;
+ }
+
+ OmLifecycleScanState.Builder scanStateBuilder = null;
+ try {
+ OmLifecycleScanState scanState = omMetadataManager.getLifecycleScanStateTable().get(bucketKey);
+ if (scanState == null || (scanState.getBucketObjID() != bucket.getObjectID() ||
+ scanState.getLifecycleConfigurationUpdateID() != policy.getUpdateID() ||
+ scanState.getScanEndTime() != null)) {
+ scanStateBuilder = new OmLifecycleScanState.Builder();
+ scanStateBuilder.setBucketKey(bucketKey);
+ scanStateBuilder.setScanStartTime(System.currentTimeMillis());
+ scanStateBuilder.setBucketObjID(bucket.getObjectID());
+ scanStateBuilder.setLifecycleConfigurationUpdateID(policy.getUpdateID());
+ LOG.info("Create/Recreate OmLifecycleScanState for {} bucket {} bucketID {} " +
+ "lifecycleConfigurationUpdateID {}", bucket.getBucketLayout(), bucketKey, bucket.getObjectID(),
+ policy.getUpdateID());
+ } else {
+ scanStateBuilder = scanState.toBuilder();
+ LOG.info("Resume OmLifecycleScanState {}", scanState);
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to get scan state for bucket {}", bucketKey, e);
+ }
+
+ try {
+ List<OmLCRule> originRuleList = policy.getRules();
+ // remove disabled rules
+ List<OmLCRule> ruleList = originRuleList.stream().filter(
+ r -> r.isEnabled()).collect(Collectors.toList());
+
+ List<OmLCRule> expirationRules = ruleList.stream()
+ .filter(r -> r.getExpiration() != null)
+ .collect(Collectors.toList());
+ List<OmLCRule> mpuRules = ruleList.stream()
+ .filter(r -> r.getAbortIncompleteMultipartUpload() != null)
+ .collect(Collectors.toList());
+
+ if (!expirationRules.isEmpty()) {
+ LimitedExpiredObjectList expiredKeyList = new LimitedExpiredObjectList(listMaxSize);
+ LimitedExpiredObjectList expiredDirList = new LimitedExpiredObjectList(listMaxSize);
+ Table<String, OmKeyInfo> keyTable = omMetadataManager.getKeyTable(bucket.getBucketLayout());
+ /**
+ * Filter treatment.
+ * "" - all objects
+ * "/" - if it's OBS/Legacy, means keys starting with "/"; If it's FSO, not supported
+ * "/key" - if it's OBS/Legacy, means keys starting with "/key", "/" is literally "/";
+ * If it's FSO, means keys or dirs starting with "key", "/" will be treated as separator mark.
+ * "key" - if it's OBS/Legacy, means keys starting with "key";
+ * if it's FSO, means keys for dirs starting with "key" too.
+ * "dir/" - if it's OBS/Legacy, means keys starting with "dir/";
+ * - if it's FSO, means keys/dirs under directory "dir", doesn't include directory "dir" itself.
+ * - For FSO bucket, as directory ModificationTime will not be updated when any of its child
+ * key/subdir changes, so remember to add the tailing slash "/" when configure prefix, otherwise
+ * the whole directory will be expired and deleted once its ModificationTime meats the condition.
+ */
+ if (bucket.getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ OmVolumeArgs volume;
+ try {
+ volume = omMetadataManager.getVolumeTable().get(omMetadataManager.getVolumeKey(bucket.getVolumeName()));
+ if (volume == null) {
+ LOG.warn("Volume {} cannot be found, might be deleted during this task's execution",
+ bucket.getVolumeName());
+ onFailure(bucketKey);
+ return result;
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to get volume {}", bucket.getVolumeName(), e);
+ onFailure(bucketKey);
+ return result;
+ }
+ evaluateFSOBucket(volume, bucket, bucketKey, keyTable, expirationRules, expiredKeyList,
+ expiredDirList, scanStateBuilder);
+ } else {
+ // use bucket name as key iterator prefix
+ evaluateBucket(bucket, keyTable, expirationRules, expiredKeyList, scanStateBuilder);
+ }
+
+ if (expiredKeyList.isEmpty() && expiredDirList.isEmpty()) {
+ LOG.info("No expired keys/dirs found/remained for bucket {}", bucketKey);
+ sendSaveScanStateRequest(scanStateBuilder, true);
+ } else {
+ LOG.info("{} expired keys and {} expired dirs found and remained for bucket {}",
+ expiredKeyList.size(), expiredDirList.size(), bucketKey);
+
+ // If trash is enabled, move files to trash, instead of send delete requests.
+ // OBS bucket doesn't support trash.
+ if (bucket.getBucketLayout() == OBJECT_STORE) {
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(),
+ bucket.getOwner(), expiredKeyList, false, scanStateBuilder, true);
+ } else {
+ // handle keys first, then directories
+ handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, true);
+ handleAndClearFullList(bucket, expiredDirList, true, scanStateBuilder, true);
+ }
+ }
+ }
+
+ if (!mpuRules.isEmpty()) {
+ processMultipartUploads(bucket, mpuRules);
+ }
+ } catch (Throwable e) {
+ LOG.error("Failed to evaluate lifecycle configuration for bucket {}", bucketKey, e);
+ onFailure(bucketKey);
+ return result;
+ }
+
+ onSuccess(bucketKey);
+ }
+
+ // By design, no one cares about the results of this call back.
+ return result;
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ private void evaluateFSOBucket(OmVolumeArgs volume, OmBucketInfo bucket, String bucketKey,
+ Table<String, OmKeyInfo> keyTable, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList expiredKeyList, LimitedExpiredObjectList expiredDirList,
+ OmLifecycleScanState.Builder scanStateBuilder) {
+ List<OmLCRule> prefixRuleList =
+ ruleList.stream().filter(r -> r.isPrefixEnable()).collect(Collectors.toList());
+ // r.isPrefixEnable() == false means empty filter
+ List<OmLCRule> noPrefixRuleList =
+ ruleList.stream().filter(r -> !r.isPrefixEnable()).collect(Collectors.toList());
+
+ if (!noPrefixRuleList.isEmpty()) {
+ // evaluate all rules against each key
+ prefixRuleList.addAll(noPrefixRuleList);
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, "",
+ prefixRuleList, expiredKeyList, expiredDirList, scanStateBuilder);
+ return;
+ }
+
+ List<RuleListWithDirectoryList> unionPrefixRuleList =
+ getRuleUnion(volume.getObjectID(), bucket, prefixRuleList, bucketKey);
+
+ if (unionPrefixRuleList != null) {
+ if (unionPrefixRuleList.isEmpty()) {
+ // fallback to evaluate the whole bucket
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, "",
+ prefixRuleList, expiredKeyList, expiredDirList, scanStateBuilder);
+ } else {
+ for (RuleListWithDirectoryList ruleWithDirList : unionPrefixRuleList) {
+ List<OmLCRule> rules = ruleWithDirList.getRuleList();
+ DirectoryList dir = ruleWithDirList.getDirList();
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, dir.getLastSubDirPath(),
+ dir.getLastSubDir(), dir.getLastSubDirKey(),
+ rules, expiredKeyList, expiredDirList, scanStateBuilder);
+ }
+ }
+ }
+ }
+
+ /**
+ * Finds the directory union list from a list of prefixes and sorts them
+ * according to the FSO depth-first iteration order.
+ */
+ private List<RuleListWithDirectoryList> getRuleUnion(long volumeId, OmBucketInfo bucket,
+ List<OmLCRule> rules, String bucketKey) {
+
+ if (rules.isEmpty() || rules.stream().anyMatch(
+ r -> r.getEffectivePrefix() == null || r.getEffectivePrefix().isEmpty())) {
+ // The union of anything with the root is just the root itself.
+ return new ArrayList();
+ }
+
+ List<RuleListWithDirectoryList> effectiveRuleList = new ArrayList<>();
+ for (OmLCRule rule : rules) {
+ String prefix = rule.getEffectivePrefix();
+ // Resolve each prefix to actual FSO directories in the DB
+ try {
+ if (!prefix.endsWith(OzoneConsts.OM_KEY_PREFIX)) {
+ // FSO bucket doesn't allow prefix without tailing '/'
+ // Prefix ends with a slash, it explicitly refers to a directory (e.g. "log/")
+ LOG.warn("Skip rule {} since FILE_SYSTEM_OPTIMIZED bucket prefix must end with '/'", rule);
+ continue;
+ }
+
+ // Normalize by removing the trailing slash for uniform comparison
+ String normalizedPrefix = prefix.substring(0, prefix.length() - 1);
+ DirectoryList dirList = getDirList(volumeId, bucket, normalizedPrefix, bucketKey);
+ // If the prefix is log/, and "log" dir really exists, then the matched dir is "log".
+ // Otherwise, this rule doesn't match any dir/file in this FSO bucket, this rule can be skipped.
+ if (!dirList.isEmpty() && dirList.isAllResolvedPrefix()) {
+ RuleListWithDirectoryList ruleListWithDirectoryList = new RuleListWithDirectoryList(
+ Collections.singletonList(rule), dirList, prefix);
+ effectiveRuleList.add(ruleListWithDirectoryList);
+ }
+ } catch (IOException e) {
+ // Directory doesn't exist or IO error, skip this rule
+ LOG.warn("Skip to evaluate rule {} due to failed to resolve prefix {} for bucket {}",
+ rule, prefix, bucketKey, e);
+ }
+ }
+
+ if (effectiveRuleList.isEmpty()) {
+ // there is no valid rule found, either prefix doesn't end with "/",
+ // or any directory along the prefix cannot be found.
+ LOG.warn("Prefix of all rules of bucket {} cannot be resolved to an existing directory. ", bucketKey);
+ return null;
+ }
+
+ if (effectiveRuleList.size() == 1) {
+ return effectiveRuleList;
+ }
+
+ // Find if one rule's prefix is the sub string of another rule's prefix.
+ // e.g.
+ // dir1/dir2/, dir1/dir2/dir3/, dir1/ -> dir1/
+ // dir1/dir2/, dir1/dir3/, dir1/dir4/ -> dir1/dir2/, dir1/dir3/, dir1/dir4dir1/
+ // dir1/dir2/dir3/, dir1/dir2/, dir2/ -> dir1/dir2/, dir2/
+ // dir1/dir2/, dir1/dir3/, dir1/ -> dir1/
+ List<RuleListWithDirectoryList> consolidatedRules = new ArrayList<>();
+ Set<OmLCRule> skipEvaluatedRuleList = new HashSet<>();
+ for (int i = 0; i < effectiveRuleList.size(); i++) {
+ OmLCRule rule = effectiveRuleList.get(i).getRuleList().get(0);
+ if (skipEvaluatedRuleList.contains(rule)) {
+ continue;
+ }
+
+ RuleListWithDirectoryList consolidatedCandidate = new RuleListWithDirectoryList();
+ String consolidatedPrefix = effectiveRuleList.get(i).getConsolidatedPrefix();
+ String finalRuleIndexID = rule.getId();
+ DirectoryList finalDirList = effectiveRuleList.get(i).getDirList();
+ for (int j = i + 1; j < effectiveRuleList.size(); j++) {
+ OmLCRule otherRule = effectiveRuleList.get(j).getRuleList().get(0);
+ if (skipEvaluatedRuleList.contains(otherRule)) {
+ continue;
+ }
+
+ DirectoryList otherDirList = effectiveRuleList.get(j).getDirList();
+ String otherPrefix = otherRule.getEffectivePrefix();
+ if (otherPrefix.startsWith(consolidatedPrefix)) {
+ LOG.info("Rule {}'s prefix {} is sub string of rule {}'s prefix {}. " +
+ " Consolidate {} into {}.", otherRule.getId(), otherPrefix, finalRuleIndexID,
+ consolidatedPrefix, otherRule.getId(), finalRuleIndexID);
+ consolidatedCandidate.addRule(otherRule);
+ skipEvaluatedRuleList.add(otherRule);
+ } else if (consolidatedPrefix.startsWith(otherPrefix)) {
+ LOG.info("Rule {}'s prefix {} is sub string of rule {}'s prefix {}. Consolidate {} int {}. ",
+ consolidatedPrefix, consolidatedPrefix, otherRule.getId(), otherPrefix, consolidatedPrefix,
+ otherRule.getId());
+ consolidatedPrefix = otherPrefix;
+ finalRuleIndexID = otherRule.getId();
+ finalDirList = otherDirList;
+ consolidatedCandidate.addRule(otherRule);
+ skipEvaluatedRuleList.add(otherRule);
+ }
+ }
+
+ consolidatedCandidate.addRule(rule);
+ consolidatedCandidate.setDirList(finalDirList);
+ consolidatedCandidate.setConsolidatedPrefix(consolidatedPrefix);
+ consolidatedRules.add(consolidatedCandidate);
+ }
+
+ // Sort the list of paths lexicographically.
+ // FSO Depth-First Search order evaluates directories in lexicographical order
+ // (since it retrieves entries from RocksDB sorted by name within the same parent).
+ // Standard string sort on logical paths separated by "/" perfectly matches this DFS order.
+ List<RuleListWithDirectoryList> sortedConsolidatedRules =
+ consolidatedRules.stream().sorted(new RuleListWithDirectoryListOrder()).collect(Collectors.toList());
+
+ LOG.info("Final consolidated rules: " +
+ sortedConsolidatedRules.stream().map(RuleListWithDirectoryList::toString).collect(Collectors.joining(", ")));
+ if (test) {
+ consolidatedRuleList = sortedConsolidatedRules;
+ }
+ return sortedConsolidatedRules;
+ }
+
+ private boolean canSkipDir(String currentDirPath, String lastScannedDirInState) {
+ if (lastScannedDirInState == null || currentDirPath.isEmpty()) {
+ return false;
+ }
+ String[] cur = currentDirPath.split(OM_KEY_PREFIX);
+ String[] last = lastScannedDirInState.split(OM_KEY_PREFIX);
+ int n = Math.min(cur.length, last.length);
+ for (int i = 0; i < n; i++) {
+ int cmp = cur[i].compareTo(last[i]);
+ if (cmp != 0) {
+ // current name > last name -> skip
+ return cmp > 0;
+ }
+ }
+ return false;
+ }
+
+ @SuppressWarnings({"checkstyle:parameternumber", "checkstyle:MethodLength"})
+ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table<String, OmKeyInfo> keyTable,
+ String directoryPath, @Nullable OmDirectoryInfo dir, String dirKey, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList keyList, LimitedExpiredObjectList dirList,
+ OmLifecycleScanState.Builder scanStateBuilder) {
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ LimitedSizeStack stack = new LimitedSizeStack(cachedDirMaxCount);
+ String lastScannedDirInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDir();
+ String lastScannedDirKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDirKey();
+ String lastScannedKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedKey();
+ try {
+ if (dir != null) {
+ stack.push(new PendingEvaluateDirectory(dir, dirKey, directoryPath, null));
+ } else {
+ // put a placeholder PendingEvaluateDirectory to stack for bucket
+ stack.push(new PendingEvaluateDirectory(null, "", "", null));
+ }
+ } catch (CapacityFullException e) {
+ LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, directoryPath != null ? directoryPath : "", e);
+ return;
+ }
+
+ HashSet<Long> deletedDirSet = new HashSet<>();
+ while (!stack.isEmpty()) {
+ if (!shouldRun()) {
+ LOG.info("LifecycleActionTask for bucket {} stopping. " +
+ "Service enabled: {}, suspended: {}, leader ready: {}",
+ bucketName, isServiceEnabled.get(), suspended.get(),
+ getOzoneManager() != null ? getOzoneManager().isLeaderReady() : "N/A");
+ return;
+ }
+
+ PendingEvaluateDirectory item = stack.pop();
+ OmDirectoryInfo currentDir = item.getDirectoryInfo();
+ String currentDirPath = item.getDirPath();
+ long currentDirObjID = currentDir == null ? bucket.getObjectID() : currentDir.getObjectID();
+ String currentDirTableKey = item.getDirTableKey();
+
+ /**
+ * /
+ * dir1 dir2 dir3 dir30
+ * / \ / \
+ * dir4 dir5 dir6 dir7
+ * / \
+ * dir8 dir9
+ * lastScannedDir = dir3/dir6/dir8, which means
+ * Scanned:
+ * dir30
+ * dir3/dir7
+ * dir3/dir6/dir9
+ * Half scanned:
+ * dir3/dir6/dir8
+ * Not scanned:
+ * dir3/dir6
+ * dir3
+ * dir1/dir5
+ * dir/dir4
+ * dir1
+ * directoryTable table key format : /volumeId/bucketId/parentId/dirName
+ * based on the depth first evaluation order, and stack push posh iteration pattern
+ * - dir1, on grand level of lastScannedDir, and name order < lastScannedDir grand, not scanned
+ * - dir2, on grand level of lastScannedDir, and name order < lastScannedDir grand, not scanned
+ * - dir3, grand of lastScannedDir, not scanned
+ * - dir30, on grand level of lastScannedDir, and name order > lastScannedDir grand, scanned, skip
+ * - dir3/dir6, parent of lastScannedDir, not scanned
+ * - dir3/dir7, parent level of lastScannedDir, and name order > lastScannedDir parent, scanned, skip
+ * - dir3/dir8, lastScannedDir, partially scanned,
+ * - dir3/dir9, same the same parentID as lastScannedDir, and name order > lastScannedDir, scanned, skip
+ */
+ if (canSkipDir(currentDirPath, lastScannedDirInState)) {
+ LOG.info("Skip {} in LifecycleActionTask for bucket {}. ", currentDirPath, bucketName);
+ continue;
+ }
+
+ lastScannedDir = currentDirPath;
+ lastScannedDirKey = currentDirTableKey;
+ if (shouldSaveState()) {
+ flushAndSaveState(bucket, keyList, dirList, scanStateBuilder);
+ }
+
+ // use current directory's object ID to iterate the keys and directories under it
+ String prefix =
+ OM_KEY_PREFIX + volumeObjId + OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX + currentDirObjID;
+ LOG.debug("Prefix {} for {}/{}", prefix, bucket.getVolumeName(), bucket.getBucketName());
+
+ // get direct sub directories
+ DirectoryList subDirSummary;
+ boolean newSubDirPushed = false;
+ long deletedDirCount = 0;
+ if (item.isFirstEvaluate()) {
+ try {
+ subDirSummary = getSubDirectory(currentDirObjID, prefix, omMetadataManager);
+ } catch (IOException e) {
+ // log failure, continue to process other directories in stack
+ LOG.warn("Failed to get sub directories of {} under {}/{}", currentDirPath, volumeName, bucketName, e);
+ continue;
+ }
+ } else {
+ // this item is a parent directory, check how many sub directories are deleted.
+ subDirSummary = item.getSubDirSummary();
+ for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) {
+ if (deletedDirSet.remove(subDir.getObjectID())) {
+ deletedDirCount++;
+ }
+ }
+ }
+
+ if (item.isFirstEvaluate()) {
+ // filter sub directory list
+ if (!subDirSummary.getSubDirList().isEmpty()) {
+ Iterator<OmDirectoryInfo> iterator = subDirSummary.getSubDirList().iterator();
+ while (iterator.hasNext()) {
+ OmDirectoryInfo subDir = iterator.next();
+ String subDirPath = currentDirPath.isEmpty() ? subDir.getName() :
+ currentDirPath + OM_KEY_PREFIX + subDir.getName();
+ if (subDirPath.startsWith(TRASH_PREFIX)) {
+ iterator.remove();
+ continue;
+ }
+ boolean matched = false;
+ for (OmLCRule rule : ruleList) {
+ if (rule.getEffectivePrefix() != null && subDirPath.startsWith(rule.getEffectivePrefix())) {
+ matched = true;
+ break;
+ }
+ }
+ if (!matched) {
+ iterator.remove();
+ }
+ }
+ }
+
+ if (!subDirSummary.getSubDirList().isEmpty()) {
+ item.setDirectoryList(subDirSummary);
+ item.setFirstEvaluate(false);
+ try {
+ stack.push(item);
+ } catch (CapacityFullException e) {
+ LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, currentDirPath, e);
+ return;
+ }
+
+ // depth first evaluation, push subDirs into stack
+ for (int i = 0; i < subDirSummary.getSubDirCount(); i++) {
+ OmDirectoryInfo subDir = subDirSummary.getSubDirList().get(i);
+ String subDirPath = currentDirPath.isEmpty() ? subDir.getName() :
+ currentDirPath + OM_KEY_PREFIX + subDir.getName();
+ try {
+ stack.push(new PendingEvaluateDirectory(subDir, subDirSummary.getSubDirKeyList().get(i),
+ subDirPath, null));
+ } catch (CapacityFullException e) {
+ LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, subDirPath, e);
+ return;
+ }
+ }
+ newSubDirPushed = true;
+ }
+ }
+
+ if (newSubDirPushed) {
+ continue;
+ }
+
+ // evaluate direct files, first check cache, then check table
+ // there are three cases:
+ // a. key is deleted in cache, while it's not deleted in table yet
+ // b. key is new added in cache, not in table yet
+ // c. key is updated in cache(rename), but not updated in table yet
+ // in this case, the fromKey is a deleted key in cache, and the toKey is a newly added key in cache,
+ // and fromKey is also in table
+ long numKeysUnderDir = 0;
+ long numKeysExpired = 0;
+ HashSet<String> deletedKeySetInCache = new HashSet();
+ HashSet<String> keySetInCache = new HashSet();
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter = keyTable.cacheIterator();
+ while (cacheIter.hasNext()) {
+ Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry = cacheIter.next();
+ OmKeyInfo key = entry.getValue().getCacheValue();
+ if (key == null) {
+ deletedKeySetInCache.add(entry.getKey().getCacheKey());
+ continue;
+ }
+ if (key.getParentObjectID() == currentDirObjID) {
+ numKeysUnderDir++;
+ keySetInCache.add(entry.getKey().getCacheKey());
+ String keyPath = currentDirPath.isEmpty() ? key.getKeyName() :
+ currentDirPath + OM_KEY_PREFIX + key.getKeyName();
+ for (OmLCRule rule : ruleList) {
+ if (rule.match(key, keyPath)) {
+ // mark key as expired, check next key
+ if (keyList.isFull()) {
+ // if keyList is full, send delete/rename request for expired keys
+ handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false);
+ }
+ keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID());
+ numKeysExpired++;
+ break;
+ }
+ }
+ lastScannedKey = entry.getKey().getCacheKey();
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyTblItr =
+ keyTable.iterator(prefix)) {
+ boolean seekPerformed = false;
+ if (lastScannedDirKeyInState != null && lastScannedDirKeyInState.compareTo(currentDirTableKey) == 0 &&
+ lastScannedKeyInState != null && lastScannedKeyInState.startsWith(prefix)) {
+ LOG.info("Seek to key {} under directory {}", scanStateBuilder.getLastScannedKey(), lastScannedDirInState);
+ keyTblItr.seek(scanStateBuilder.getLastScannedKey());
+ seekPerformed = true;
+ }
+
+ while (keyTblItr.hasNext()) {
+ if (shouldSaveState()) {
+ LOG.info("Saving scan state for bucket {} at key {}", bucketName, lastScannedKey);
+ flushAndSaveState(bucket, keyList, dirList, scanStateBuilder);
+ }
+ Table.KeyValue<String, OmKeyInfo> keyValue = keyTblItr.next();
+ OmKeyInfo key = keyValue.getValue();
+ String keyPath = currentDirPath.isEmpty() ? key.getKeyName() :
+ currentDirPath + OM_KEY_PREFIX + key.getKeyName();
+ if (seekPerformed && keyValue.getKey().equals(scanStateBuilder.getLastScannedKey())) {
+ continue;
+ }
+ if (deletedKeySetInCache.remove(keyValue.getKey()) || keySetInCache.remove(keyValue.getKey())) {
+ continue;
+ }
+ numKeyIterated++;
+ numKeysUnderDir++;
+ for (OmLCRule rule : ruleList) {
+ if (key.getParentObjectID() == currentDirObjID && rule.match(key, keyPath)) {
+ // mark key as expired, check next key
+ if (keyList.isFull()) {
+ // if keyList is full, send delete request for pending deletion keys
+ handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false);
+ }
+ keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID());
+ numKeysExpired++;
+ break;
+ }
+ }
+ lastScannedKey = keyValue.getKey();
+ }
+ } catch (IOException e) {
+ // log failure and continue the process other directories in stack
+ LOG.warn("Failed to iterate keyTable for bucket {}/{}", volumeName, bucketName, e);
+ continue;
+ }
+
+ // if this directory is empty or all files/subDirs are expired, evaluate itself
+ if ((numKeysUnderDir == 0 && subDirSummary.getSubDirCount() == 0) ||
+ (numKeysUnderDir == numKeysExpired && deletedDirCount == subDirSummary.getSubDirCount())) {
+ List<String> pathList = new ArrayList<>();
+ boolean skipDir = false;
+ for (int i = 0; i < ruleList.size(); i++) { // NOPMD
+ OmLCRule rule = ruleList.get(i);
+ String path = rule.getEffectivePrefix() != null && rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX) ?
+ currentDirPath + OM_KEY_PREFIX : currentDirPath;
+ if (path != null && path.equals(rule.getEffectivePrefix())) {
+ LOG.info("Prefix directory {} doesn't get expired", path);
+ skipDir = true;
+ break;
+ }
+ pathList.add(path);
+ }
+
+ if (skipDir) {
+ continue;
+ }
+ for (int i = 0; i < ruleList.size(); i++) {
+ String path = pathList.get(i);
+ OmLCRule rule = ruleList.get(i);
+ if (currentDir != null && rule.match(currentDir, path)) {
+ if (dirList.isFull()) {
+ // if expiredDirList is full, send delete request for both pending deletion keys and directories
+ handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false);
+ handleAndClearFullList(bucket, dirList, true, scanStateBuilder, false);
+ if (getInjector(2) != null && getInjector(2).getException() != null) {
+ Throwable ex = getInjector(2).getException();
+ getInjector(2).setException(null);
+ throw new RuntimeException(ex);
+ }
+ }
+ dirList.add(currentDirPath, 0, currentDir.getUpdateID());
+ deletedDirSet.add(currentDir.getObjectID());
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ private DirectoryList getSubDirectory(long dirObjID, String prefix, OMMetadataManager metaMgr)
+ throws IOException {
+ DirectoryList subDirList = new DirectoryList();
+
+ // Check all dirTable cache for any sub paths.
+ Table dirTable = metaMgr.getDirectoryTable();
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>>>
+ cacheIter = dirTable.cacheIterator();
+ HashSet<String> deletedDirSet = new HashSet();
+ while (cacheIter.hasNext()) {
+ Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>> entry =
+ cacheIter.next();
+ numDirIterated++;
+ OmDirectoryInfo cacheOmDirInfo = entry.getValue().getCacheValue();
+ if (cacheOmDirInfo == null) {
+ deletedDirSet.add(entry.getKey().getCacheKey());
+ continue;
+ }
+ if (cacheOmDirInfo.getParentObjectID() == dirObjID) {
+ subDirList.addSubDir(entry.getKey().getCacheKey(), cacheOmDirInfo, cacheOmDirInfo.getName());
+ }
+ }
+
+ // Check dirTable entries for any sub paths.
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmDirectoryInfo>>
+ iterator = dirTable.iterator(prefix)) {
+ while (iterator.hasNext()) {
+ numDirIterated++;
+ Table.KeyValue<String, OmDirectoryInfo> entry = iterator.next();
+ OmDirectoryInfo dir = entry.getValue();
+ if (deletedDirSet.contains(entry.getKey())) {
+ continue;
+ }
+ if (dir.getParentObjectID() == dirObjID) {
+ subDirList.addSubDir(entry.getKey(), dir, dir.getName());
+ }
+ }
+ }
+ return subDirList;
+ }
+
+ private void flushAndSaveState(OmBucketInfo bucket, LimitedExpiredObjectList expiredKeyList,
+ LimitedExpiredObjectList expiredDirList, OmLifecycleScanState.Builder scanStateBuilder) {
+ boolean saved = false;
+ if (expiredKeyList != null && !expiredKeyList.isEmpty()) {
+ if (bucket.getBucketLayout() == OBJECT_STORE) {
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(),
+ bucket.getOwner(), expiredKeyList, false, scanStateBuilder, false);
+ } else {
+ handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, false);
+ }
+ saved = true;
+ }
+ if (expiredDirList != null && !expiredDirList.isEmpty()) {
+ if (bucket.getBucketLayout() != OBJECT_STORE) {
+ handleAndClearFullList(bucket, expiredDirList, true, scanStateBuilder, false);
+ saved = true;
+ }
+ }
+ if (!saved) {
+ sendSaveScanStateRequest(scanStateBuilder, false);
+ }
+ lastStateSaveTime = Time.monotonicNow();
+ lastStateSaveKeyCount = numKeyIterated;
+ }
+
+ private void evaluateBucket(OmBucketInfo bucketInfo,
+ Table<String, OmKeyInfo> keyTable, List<OmLCRule> ruleList, LimitedExpiredObjectList expiredKeyList,
+ OmLifecycleScanState.Builder scanStateBuilder) {
+ String volumeName = bucketInfo.getVolumeName();
+ String bucketName = bucketInfo.getBucketName();
+ String bucketPrefix = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyTblItr =
+ keyTable.iterator(bucketPrefix)) {
+ boolean seekPerformed = false;
+ if (scanStateBuilder != null && scanStateBuilder.getLastScannedKey() != null) {
+ keyTblItr.seek(scanStateBuilder.getLastScannedKey());
+ seekPerformed = true;
+ }
+
+ while (keyTblItr.hasNext()) {
+ if (!shouldRun()) {
+ LOG.info("KeyLifecycleService is suspended or disabled. " +
+ "Stopping LifecycleActionTask for bucket {}.", bucketName);
+ return;
+ }
+ if (shouldSaveState()) {
+ flushAndSaveState(bucketInfo, expiredKeyList, null, scanStateBuilder);
+ }
+ Table.KeyValue<String, OmKeyInfo> keyValue = keyTblItr.next();
+ if (seekPerformed && keyValue.getKey().equals(scanStateBuilder.getLastScannedKey())) {
+ continue;
+ }
+ processKey(bucketInfo, keyValue.getValue(), ruleList, expiredKeyList, scanStateBuilder);
+ numKeyIterated++;
+ lastScannedKey = keyValue.getKey();
+ }
+ } catch (IOException e) {
+ // log failure and continue the process to delete/move files already identified in this run
+ LOG.warn("Failed to iterate through bucket {}/{}", volumeName, bucketName, e);
+ }
+ }
+
+ private void processKey(OmBucketInfo bucketInfo, OmKeyInfo key, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList expiredKeyList, OmLifecycleScanState.Builder scanStateBuilder) {
+ if (bucketInfo.getBucketLayout() == BucketLayout.LEGACY &&
+ key.getKeyName().startsWith(TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)) {
+ return;
+ }
+ for (OmLCRule rule : ruleList) {
+ if (rule.match(key)) {
+ // mark key as expired, check next key
+ if (expiredKeyList.isFull()) {
+ // if expiredKeyList is full, send delete/rename request for expired keys
+ handleAndClearFullList(bucketInfo, expiredKeyList, false, scanStateBuilder, false);
+ if (getInjector(2) != null && getInjector(2).getException() != null) {
+ Throwable ex = getInjector(2).getException();
+ getInjector(2).setException(null);
+ throw new RuntimeException(ex);
+ }
+ }
+ expiredKeyList.add(key.getKeyName(), key.getReplicatedSize(), key.getUpdateID());
+ break;
+ }
+ }
+ }
+
+ /**
+ * Process AbortIncompleteMultipartUpload actions for incomplete multipart uploads.
+ * Iterates through the multipartInfoTable and aborts uploads that match the rule criteria
+ * and have exceeded the configured days after initiation.
+ *
+ * @param bucketInfo the bucket information
+ * @param ruleList list of lifecycle rules with AbortIncompleteMultipartUpload action
+ */
+ private void processMultipartUploads(OmBucketInfo bucketInfo, List<OmLCRule> ruleList) {
+ String volumeName = bucketInfo.getVolumeName();
+ String bucketName = bucketInfo.getBucketName();
+ String bucketPrefix = omMetadataManager.getBucketKeyPrefix(volumeName, bucketName);
+
+ LOG.debug("Processing AbortIncompleteMultipartUpload actions for bucket {}/{}", volumeName, bucketName);
+
+ PartCountLimitedList expiredUploads = new PartCountLimitedList(mpuAbortLimitPerTask);
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmMultipartKeyInfo>> mpuIterator =
+ omMetadataManager.getMultipartInfoTable().iterator(bucketPrefix)) {
+ while (mpuIterator.hasNext()) {
+ if (!shouldRun()) {
+ LOG.info("KeyLifecycleService is suspended or disabled. " +
+ "Stopping multipart upload processing for bucket {}.", bucketName);
+ return;
+ }
+ Table.KeyValue<String, OmMultipartKeyInfo> entry = mpuIterator.next();
+ OmMultipartKeyInfo mpuKeyInfo = entry.getValue();
+ numMultipartUploadIterated++;
+
+ OmMultipartUpload upload;
+ try {
+ upload = OmMultipartUpload.from(entry.getKey());
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Failed to parse multipart upload key {} in bucket {}/{}, skipping",
+ entry.getKey(), volumeName, bucketName, e);
+ continue;
+ }
+
+ upload.setCreationTime(Instant.ofEpochMilli(mpuKeyInfo.getCreationTime()));
+ String keyName = upload.getKeyName();
+
+ OmLCRule matchingRule = null;
+ OmKeyInfo openKeyInfo = null;
+ boolean openKeyFetchAttempted = false;
+ boolean skipUpload = false;
+ for (OmLCRule rule : ruleList) {
+ if (!passesAgeAndPrefix(upload, keyName, rule)) {
+ continue;
+ }
+ if (!rule.isTagEnable()) {
+ matchingRule = rule;
+ break;
+ }
+ if (!openKeyFetchAttempted) {
+ openKeyFetchAttempted = true;
+ try {
+ String multipartOpenKey = OMMultipartUploadUtils.getMultipartOpenKey(
+ volumeName, bucketName, keyName, upload.getUploadId(),
+ omMetadataManager, bucketInfo.getBucketLayout());
+ openKeyInfo = omMetadataManager.getOpenKeyTable(
+ bucketInfo.getBucketLayout()).get(multipartOpenKey);
+ } catch (OMException e) {
+ LOG.warn("Failed to get multipart open key for {}/{}/{}, skipping",
+ volumeName, bucketName, keyName, e);
+ skipUpload = true;
+ break;
+ } catch (IOException e) {
+ LOG.warn("Failed to read open key table for {}/{}/{}, skipping",
+ volumeName, bucketName, keyName, e);
+ skipUpload = true;
+ break;
+ }
+ if (openKeyInfo == null) {
+ LOG.debug("Orphan multipart upload {}/{}/{} has no open key entry, skipping tag-requiring rules",
+ volumeName, bucketName, keyName);
+ }
+ }
+ if (openKeyInfo == null) {
+ continue;
+ }
+ OmLCFilter filter = rule.getFilter();
+ if (filter == null || filter.match(openKeyInfo, keyName)) {
+ matchingRule = rule;
+ break;
+ }
+ }
+
+ if (skipUpload || matchingRule == null) {
+ continue;
+ }
+
+ if (expiredUploads.isFull()) {
+ LOG.info("Multipart upload batch reached part count limit {}, aborting current batch " +
+ "({} uploads, {} parts) for bucket {}/{}",
+ mpuAbortLimitPerTask, expiredUploads.size(), expiredUploads.getPartCount(),
+ volumeName, bucketName);
+ abortExpiredMultipartUploadsAndClear(bucketInfo, expiredUploads);
+ }
+
+ // Split-schema MPUs keep parts in multipartPartsTable (the embedded map
+ // is empty); legacy MPUs use the embedded map. An MPU with no uploaded
+ // parts is valid (S3 allows aborting it with an empty parts list).
+ int uploadedParts;
+ try {
+ uploadedParts = mpuKeyInfo.getSchemaVersion()
+ == OmMultipartKeyInfo.SPLIT_PARTS_TABLE_SCHEMA_VERSION
+ ? OMMultipartUploadUtils.countParts(omMetadataManager, upload.getUploadId())
+ : mpuKeyInfo.getPartKeyInfoMap().size();
+ } catch (IOException e) {
+ LOG.warn("Failed to count parts for MPU {}/{}/{} uploadId {}, skipping",
+ volumeName, bucketName, keyName, upload.getUploadId(), e);
+ continue;
+ }
+ expiredUploads.add(upload, uploadedParts);
+ LOG.debug("Multipart upload {}/{}/{} with uploadId {} ({} parts) will be aborted",
+ volumeName, bucketName, keyName, upload.getUploadId(), uploadedParts);
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to iterate multipartInfoTable for bucket {}/{}", volumeName, bucketName, e);
+ return;
+ }
+
+ if (!expiredUploads.isEmpty()) {
+ LOG.info("{} expired multipart uploads ({} parts) remaining for bucket {}/{}",
+ expiredUploads.size(), expiredUploads.getPartCount(), volumeName, bucketName);
+ abortExpiredMultipartUploadsAndClear(bucketInfo, expiredUploads);
+ }
+ }
+
+ /**
+ * Returns true if the upload passes the age and prefix checks for the given rule,
+ * without consulting the open key table (no tag evaluation).
+ */
+ private boolean passesAgeAndPrefix(OmMultipartUpload upload, String keyName, OmLCRule rule) {
+ if (!rule.getAbortIncompleteMultipartUpload().shouldAbort(
+ upload.getCreationTime().toEpochMilli())) {
+ return false;
+ }
+ String effectivePrefix = rule.getEffectivePrefix();
+ return effectivePrefix == null || keyName.startsWith(effectivePrefix);
+ }
+
+ /**
+ * Abort expired multipart uploads by sending an abort request.
+ *
+ * @param bucketInfo the bucket information
+ * @param expiredUploads list of expired multipart uploads to abort
+ */
+ private void abortExpiredMultipartUploads(OmBucketInfo bucketInfo, List<OmMultipartUpload> expiredUploads) {
+ String volumeName = bucketInfo.getVolumeName();
+ String bucketName = bucketInfo.getBucketName();
+
+ List<OzoneManagerProtocolProtos.ExpiredMultipartUploadInfo> expiredMPUInfoList = expiredUploads.stream()
+ .map(upload -> OzoneManagerProtocolProtos.ExpiredMultipartUploadInfo.newBuilder()
+ .setName(upload.getDbKey())
+ .build())
+ .collect(Collectors.toList());
+
+ OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket expiredMPUBucket =
+ OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .addAllMultipartUploads(expiredMPUInfoList)
+ .build();
+
+ OzoneManagerProtocolProtos.MultipartUploadsExpiredAbortRequest abortRequest =
+ OzoneManagerProtocolProtos.MultipartUploadsExpiredAbortRequest.newBuilder()
+ .addExpiredMultipartUploadsPerBucket(expiredMPUBucket)
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.AbortExpiredMultiPartUploads)
+ .setMultipartUploadsExpiredAbortRequest(abortRequest)
- .setVersion(ClientVersion.CURRENT_VERSION)
++ .setVersion(ClientVersion.CURRENT.serialize())
+ .setClientId(clientId.toString())
+ .build();
+
+ try {
+ long startTime = System.nanoTime();
+ OzoneManagerProtocolProtos.OMResponse response = OzoneManagerRatisUtils.submitRequest(
+ getOzoneManager(), omRequest, clientId, callId.getAndIncrement());
+ long endTime = System.nanoTime();
+
+ if (response != null) {
+ if (response.getSuccess()) {
+ numMultipartUploadAborted += expiredUploads.size();
+
+ LOG.info("Successfully aborted {} multipart uploads for bucket {}/{} in {} ns",
+ expiredUploads.size(), volumeName, bucketName, endTime - startTime);
+ } else {
+ LOG.error("Failed to abort multipart uploads for bucket {}/{}: {}",
+ volumeName, bucketName, response.getMessage());
+ }
+ } else {
+ LOG.error("Received null response when aborting multipart uploads for bucket {}/{}",
+ volumeName, bucketName);
+ }
+ } catch (ServiceException e) {
+ LOG.error("Failed to submit abort multipart uploads request for bucket {}/{}",
+ volumeName, bucketName, e);
+ }
+ }
+
+ private void abortExpiredMultipartUploadsAndClear(OmBucketInfo bucketInfo,
+ PartCountLimitedList expiredUploads) {
+ if (expiredUploads.isEmpty()) {
+ return;
+ }
+
+ abortExpiredMultipartUploads(bucketInfo, expiredUploads.getUploads());
+ expiredUploads.clear();
+ }
+
+ /**
+ * If the prefix is /dir1/dir2, but dir1 doesn't exist, then it will return an exception.
+ * If the prefix is /dir1/dir2, but dir2 doesn't exist, then it will return a list with dir1 only.
+ * If the prefix is /dir1/dir2, although dir1 exists, but get(dir1) failed with IOException,
+ * then it will return an exception too.
+ */
+ private DirectoryList getDirList(long volumeID, OmBucketInfo bucket, String prefix, String bucketKey)
+ throws IOException {
+ // find KeyInfo of each directory for the prefix
+ java.nio.file.Path keyPath = Paths.get(prefix);
+ Iterator<java.nio.file.Path> elements = keyPath.iterator();
+ long lastKnownParentId = bucket.getObjectID();
+ DirectoryList directoryList = new DirectoryList();
+ StringBuffer currentDirPath = new StringBuffer();
+ while (elements.hasNext()) {
+ String dirName = elements.next().toString();
+ String dbDirName = omMetadataManager.getOzonePathKey(
+ volumeID, bucket.getObjectID(), lastKnownParentId, dirName);
+ try {
+ OmDirectoryInfo omDirInfo = omMetadataManager.getDirectoryTable().get(dbDirName);
+ // It's OK there is no directory for the last part of the prefix, which is probably not a directory
+ if (omDirInfo == null) {
+ if (elements.hasNext()) {
+ throw new OMException("Directory " + dbDirName + " does not exist for bucket " + bucketKey,
+ OMException.ResultCodes.DIRECTORY_NOT_FOUND);
+ }
+ directoryList.setNotAllResolvedPrefix();
+ } else {
+ if (currentDirPath.length() == 0) {
+ currentDirPath.append(dirName);
+ } else {
+ currentDirPath.append('/').append(dirName);
+ }
+ directoryList.addSubDir(dbDirName, omDirInfo, currentDirPath.toString());
+ lastKnownParentId = omDirInfo.getObjectID();
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to get directory {} for bucket {}", dbDirName, bucketKey, e);
+ throw new IOException("Failed to get directory " + dbDirName + " for bucket " + bucketKey);
+ }
+ }
+ return directoryList;
+ }
+
+ private void onFailure(String bucketName) {
+ inFlight.remove(bucketName);
+ metrics.incrNumFailureTask();
+ metrics.incNumKeyIterated(numKeyIterated);
+ metrics.incNumDirIterated(numDirIterated);
+ metrics.incNumMultipartUploadIterated(numMultipartUploadIterated);
+ long timeSpent = Time.monotonicNow() - taskStartTime;
+ LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " +
+ "deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " +
+ "aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated,
+ numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted,
+ numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted);
+ }
+
+ private void onSuccess(String bucketName) {
+ inFlight.remove(bucketName);
+ metrics.incrNumSuccessTask();
+ long timeSpent = Time.monotonicNow() - taskStartTime;
+ metrics.incTaskLatencyMs(timeSpent);
+ metrics.incNumKeyIterated(numKeyIterated);
+ metrics.incNumDirIterated(numDirIterated);
+ metrics.incNumMultipartUploadIterated(numMultipartUploadIterated);
+ metrics.incNumMultipartUploadAborted(numMultipartUploadAborted);
+ LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " +
+ "deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " +
+ "aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated,
+ numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted,
+ numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted);
+ }
+
+ private void handleAndClearFullList(OmBucketInfo bucket, LimitedExpiredObjectList keysList,
+ boolean dir, OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
+ if (moveToTrashEnabled.get() && bucket.getBucketLayout() != OBJECT_STORE && getEffectiveOzoneTrash() != null) {
+ moveToTrash(bucket, keysList, dir);
+ sendSaveScanStateRequest(scanStateBuilder, scanFinished);
+ } else {
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(),
+ bucket.getOwner(), keysList, dir, scanStateBuilder, scanFinished);
+ }
+ }
+
+ private void sendSaveScanStateRequest(OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
+ if (scanStateBuilder != null) {
+ if (lastScannedDir != null) {
+ scanStateBuilder.setLastScannedDir(lastScannedDir);
+ }
+ if (lastScannedDirKey != null) {
+ scanStateBuilder.setLastScannedDirKey(lastScannedDirKey);
+ }
+ if (lastScannedKey != null) {
+ scanStateBuilder.setLastScannedKey(lastScannedKey);
+ }
+ if (scanFinished) {
+ scanStateBuilder.setScanEndTime(System.currentTimeMillis());
+ }
+ OmLifecycleScanState state = scanStateBuilder.build();
+
+ SaveLifecycleScanStateRequest saveRequest = SaveLifecycleScanStateRequest.newBuilder()
+ .setState(state.getProtobuf())
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState)
- .setVersion(ClientVersion.CURRENT_VERSION)
++ .setVersion(ClientVersion.CURRENT.serialize())
+ .setClientId(clientId.toString())
+ .setSaveLifecycleScanStateRequest(saveRequest)
+ .build();
+
+ LOG.debug("Save scan state {}", state);
+ try {
+ OzoneManagerRatisUtils.submitRequest(getOzoneManager(), omRequest, clientId, callId.getAndIncrement());
+ } catch (ServiceException e) {
+ LOG.error("Failed to submit SaveLifecycleScanState request", e);
+ }
+ }
+ }
+
+ private void sendDeleteKeysRequestAndClearList(String volume, String bucket, String bucketOwner,
+ LimitedExpiredObjectList keysList, boolean dir,
+ OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
+ try {
+ if (getInjector(1) != null) {
+ try {
+ getInjector(1).pause();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(bucketOwner);
+ int batchSize = keyDeleteBatchSize;
+ int startIndex = 0;
+ for (int i = 0; i < keysList.size();) {
+ DeleteKeyArgs.Builder builder =
+ DeleteKeyArgs.newBuilder().setBucketName(bucket).setVolumeName(volume);
+ int endIndex = startIndex + (batchSize < (keysList.size() - startIndex) ?
+ batchSize : keysList.size() - startIndex);
+ int keyCount = endIndex - startIndex;
+ builder.addAllKeys(keysList.nameSubList(startIndex, endIndex));
+ builder.addAllUpdateIDs(keysList.updateIDSubList(startIndex, endIndex));
+
+ DeleteKeyArgs deleteKeyArgs = builder.build();
+ DeleteKeysRequest.Builder requestBuilder = DeleteKeysRequest.newBuilder()
+ .setDeleteKeys(deleteKeyArgs)
+ .setSourceType(RequestSource.LIFECYCLE);
+
+ if (scanStateBuilder != null) {
+ if (lastScannedKey != null) {
+ scanStateBuilder.setLastScannedKey(lastScannedKey);
+ }
+ if (lastScannedDir != null) {
+ scanStateBuilder.setLastScannedDir(lastScannedDir);
+ }
+ if (lastScannedDirKey != null) {
+ scanStateBuilder.setLastScannedDirKey(lastScannedDirKey);
+ }
+ if (scanFinished) {
+ scanStateBuilder.setScanEndTime(System.currentTimeMillis());
+ }
+ OmLifecycleScanState state = scanStateBuilder.build();
+ requestBuilder.setScanState(state.getProtobuf());
+ LOG.debug("Save scan state: {}", state);
+ }
+
+ DeleteKeysRequest deleteKeysRequest = requestBuilder.build();
+ LOG.debug("request size {} for {} keys", deleteKeysRequest.getSerializedSize(), keyCount);
+
+ if (deleteKeysRequest.getSerializedSize() < ratisByteLimit) {
+ OMRequest omRequestRaw = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKeys)
- .setVersion(ClientVersion.CURRENT_VERSION)
++ .setVersion(ClientVersion.CURRENT.serialize())
+ .setClientId(clientId.toString())
+ .setDeleteKeysRequest(deleteKeysRequest)
+ .build();
+ long startTime = System.nanoTime();
+ final OzoneManagerProtocolProtos.OMResponse response;
+ try {
+ final OMClientRequest omClientRequest = OzoneManagerRatisUtils.createClientRequest(
+ omRequestRaw, getOzoneManager());
+ response = ugi.doAs(new PrivilegedExceptionAction<OzoneManagerProtocolProtos.OMResponse>() {
+ @Override
+ public OzoneManagerProtocolProtos.OMResponse run() throws Exception {
+ // perform preExecute as ratis submit does not perform preExecute
+ OMRequest omRequest = omClientRequest.preExecute(getOzoneManager());
+ return OzoneManagerRatisUtils.submitRequest(
+ getOzoneManager(), omRequest, clientId, callId.getAndIncrement());
+ }
+ });
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ long endTime = System.nanoTime();
+ LOG.debug("DeleteKeys request with {} keys cost {} ns", keyCount, endTime - startTime);
+ long deletedCount = keyCount;
+ long deletedSize = keysList.replicatedSizeSubList(startIndex, endIndex)
+ .stream().mapToLong(Long::longValue).sum();
+ if (response != null) {
+ if (!response.getSuccess()) {
+ // log the failure and continue the iterating
+ LOG.error("DeleteKeys request " + response.getStatus() + " failed for volume: {}, bucket: {}",
+ volume, bucket);
+ if (response.getDeleteKeysResponse().hasUnDeletedKeys()) {
+ DeleteKeyArgs unDeletedKeys = response.getDeleteKeysResponse().getUnDeletedKeys();
+ for (String key : unDeletedKeys.getKeysList()) {
+ Long size = keysList.getReplicatedSize(key);
+ if (size == null) {
+ LOG.error("Undeleted key {}/{}/{} doesn't in keyLists", volume, bucket, key);
+ continue;
+ }
+ deletedCount -= 1;
+ deletedSize -= size;
+ }
+ }
+ for (DeleteKeyError e : response.getDeleteKeysResponse().getErrorsList()) {
+ Long size = keysList.getReplicatedSize(e.getKey());
+ if (size == null) {
+ LOG.error("Deleted error key {}/{}/{} doesn't in keyLists", volume, bucket, e.getKey());
+ continue;
+ }
+ deletedCount -= 1;
+ deletedSize -= size;
+ }
+ } else {
+ LOG.debug("DeleteKeys request of total {} keys, {} not deleted", keyCount,
+ response.getDeleteKeysResponse().getErrorsCount());
+ }
+ }
+ if (dir) {
+ numDirDeleted += deletedCount;
+ metrics.incrNumDirDeleted(deletedCount);
+ } else {
+ numKeyDeleted += deletedCount;
+ sizeKeyDeleted += deletedSize;
+ metrics.incrNumKeyDeleted(deletedCount);
+ metrics.incrSizeKeyDeleted(deletedSize);
+ }
+ i += keyCount;
+ startIndex += keyCount;
+ } else {
+ batchSize /= 2;
+ }
+ }
+ } catch (ServiceException e) {
+ LOG.error("Failed to send DeleteKeysRequest", e);
+ } finally {
+ keysList.clear();
+ }
+ }
+
+ private void moveToTrash(OmBucketInfo bucket, LimitedExpiredObjectList keysList, boolean isDir) {
+ if (keysList.isEmpty()) {
+ return;
+ }
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ String trashRoot = TRASH_PREFIX + OM_KEY_PREFIX + bucket.getOwner();
+ Path trashCurrent = new Path(trashRoot, CURRENT);
+ try {
+ checkAndCreateTrashDirIfNeeded(bucket, trashCurrent);
+ } catch (IOException e) {
+ keysList.clear();
+ return;
+ }
+
+ for (int i = 0; i < keysList.size(); i++) {
+ String keyName = keysList.getName(i);
+ Path keyPath = new Path(OzoneConsts.OZONE_URI_DELIMITER + keyName);
+ Path baseKeyTrashPath = Path.mergePaths(trashCurrent, keyPath.getParent());
+ try {
+ checkAndCreateTrashDirIfNeeded(bucket, baseKeyTrashPath);
+ } catch (IOException e) {
+ LOG.error("Failed to check and create Trash dir {} for bucket {}/{}", baseKeyTrashPath,
+ volumeName, bucketName, e);
+ continue;
+ }
+ String targetKeyName = trashCurrent + OM_KEY_PREFIX + keyName;
+ KeyArgs keyArgs = KeyArgs.newBuilder().setKeyName(keyName)
+ .setVolumeName(volumeName).setBucketName(bucketName).build();
+
+ /**
+ * Trash examples:
+ * /s3v/test/readme -> /s3v/test/.Trash/hadoop/Current/readme
+ * /s3v/test/dir1/readme -> /s3v/test/.Trash/hadoop/Current/dir1/readme
+ */
+ RenameKeyRequest renameKeyRequest = RenameKeyRequest.newBuilder()
+ .setKeyArgs(keyArgs)
+ .setToKeyName(targetKeyName)
+ .setUpdateID(keysList.getUpdateID(i))
+ .build();
+
+ // send request out
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
- .setVersion(ClientVersion.CURRENT_VERSION)
++ .setVersion(ClientVersion.CURRENT.serialize())
+ .setClientId(clientId.toString())
+ .setRenameKeyRequest(renameKeyRequest)
+ .build();
+ try {
+ // perform preExecute as ratis submit do no perform preExecute
+ OMClientRequest omClientRequest = OzoneManagerRatisUtils.createClientRequest(omRequest, ozoneManager);
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(bucket.getOwner());
+ OzoneManagerProtocolProtos.OMResponse omResponse =
+ ugi.doAs(new PrivilegedExceptionAction<OzoneManagerProtocolProtos.OMResponse>() {
+ @Override
+ public OzoneManagerProtocolProtos.OMResponse run() throws Exception {
+ OMRequest request = omClientRequest.preExecute(ozoneManager);
+ return OzoneManagerRatisUtils.submitRequest(getOzoneManager(),
+ request, clientId, callId.getAndIncrement());
+ }
+ });
+ if (omResponse != null) {
+ if (!omResponse.getSuccess()) {
+ // log the failure and continue the iterating
+ LOG.error("RenameKey request failed with source key: {}, dest key: {}", keyName, targetKeyName);
+ continue;
+ }
+ }
+ LOG.info("RenameKey request succeed with source key: {}, dest key: {}", keyName, targetKeyName);
+
+ if (isDir) {
+ numDirRenamed += 1;
+ metrics.incrNumDirRenamed(1);
+ } else {
+ numKeyRenamed += 1;
+ sizeKeyRenamed += keysList.getReplicatedSize(i);
+ metrics.incrNumKeyRenamed(1);
+ metrics.incrSizeKeyRenamed(keysList.getReplicatedSize(i));
+ }
+ } catch (IOException | InterruptedException e) {
+ LOG.error("Failed to send RenameKeysRequest", e);
+ }
+ }
+ keysList.clear();
+ }
+
+ private void checkAndCreateTrashDirIfNeeded(OmBucketInfo bucket, Path dirPath) throws IOException {
+ OmKeyArgs key = new OmKeyArgs.Builder().setVolumeName(bucket.getVolumeName())
+ .setBucketName(bucket.getBucketName()).setKeyName(dirPath.toString())
+ .setOwnerName(bucket.getOwner()).build();
+ try {
+ ozoneManager.getFileStatus(key);
+ } catch (IOException e) {
+ if (e instanceof OMException &&
+ (((OMException) e).getResult() == OMException.ResultCodes.FILE_NOT_FOUND ||
+ ((OMException) e).getResult() == OMException.ResultCodes.DIRECTORY_NOT_FOUND)) {
+ // create the trash/Current directory for user
+ KeyArgs keyArgs = KeyArgs.newBuilder().setVolumeName(bucket.getVolumeName())
+ .setBucketName(bucket.getBucketName()).setKeyName(dirPath.toString())
+ .setOwnerName(bucket.getOwner()).setRecursive(true).build();
+ OMRequest omRequest = OMRequest.newBuilder().setCreateDirectoryRequest(
+ CreateDirectoryRequest.newBuilder().setKeyArgs(keyArgs))
+ .setCmdType(OzoneManagerProtocolProtos.Type.CreateDirectory)
- .setVersion(ClientVersion.CURRENT_VERSION)
++ .setVersion(ClientVersion.CURRENT.serialize())
+ .setClientId(clientId.toString())
+ .build();
+ try {
+ // perform preExecute as ratis submit do no perform preExecute
+ final OMClientRequest omClientRequest = OzoneManagerRatisUtils.createClientRequest(omRequest, ozoneManager);
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(bucket.getOwner());
+ OzoneManagerProtocolProtos.OMResponse omResponse =
+ ugi.doAs(new PrivilegedExceptionAction<OzoneManagerProtocolProtos.OMResponse>() {
+ @Override
+ public OzoneManagerProtocolProtos.OMResponse run() throws Exception {
+ OMRequest request = omClientRequest.preExecute(ozoneManager);
+ return OzoneManagerRatisUtils.submitRequest(getOzoneManager(),
+ request, clientId, callId.getAndIncrement());
+ }
+ });
+
+ if (omResponse != null) {
+ if (!omResponse.getSuccess()) {
+ LOG.error("CreateDirectory request failed with {}, path: {}",
+ omResponse.getMessage(), dirPath);
+ throw new IOException("Failed to create trash directory " + dirPath);
+ }
+ }
+ LOG.info("Created directory {}/{}/{}", bucket.getVolumeName(), bucket.getBucketName(), dirPath);
+ } catch (IOException | InterruptedException e1) {
+ LOG.error("Failed to send CreateDirectoryRequest for {}", dirPath, e1);
+ throw new IOException("Failed to send CreateDirectoryRequest request for " + dirPath);
+ }
+ } else {
+ LOG.error("Failed to get trash current directory {} status", dirPath, e);
+ throw e;
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public static FaultInjector getInjector(int index) {
+ return injectors != null ? injectors.get(index) : null;
+ }
+
+ @VisibleForTesting
+ public static void setInjectors(List<FaultInjector> instance) {
+ injectors = instance;
+ }
+
+ @VisibleForTesting
+ public static Logger getLog() {
+ return LOG;
+ }
+
+ @VisibleForTesting
+ public void setListMaxSize(int size) {
+ this.listMaxSize = size;
+ }
+
+ // Returns the test-injected OzoneTrash if set, otherwise the live instance
+ // from OzoneManager. This is needed because startTrashEmptier() runs after
+ // keyManager.start() in all OzoneManager startup paths, so the field cannot
+ // be populated eagerly in the constructor.
+ private OzoneTrash getEffectiveOzoneTrash() {
+ return ozoneTrash != null ? ozoneTrash : ozoneManager.getOzoneTrash();
+ }
+
+ @VisibleForTesting
+ public void setOzoneTrash(OzoneTrash ozoneTrash) {
+ this.ozoneTrash = ozoneTrash;
+ }
+
+ @VisibleForTesting
+ public void setMoveToTrashEnabled(boolean enabled) {
+ this.moveToTrashEnabled.set(enabled);
+ }
+
+ /**
+ * An in-memory list with limited size to hold expired object infos, including object name and current update ID.
+ */
+ public static class LimitedExpiredObjectList {
+ private final LimitedSizeList<String> objectNames;
+ private final List<Long> objectReplicatedSize;
+ private final List<Long> objectUpdateIDs;
+
+ public LimitedExpiredObjectList(int maxListSize) {
+ this.objectNames = new LimitedSizeList<>(maxListSize);
+ this.objectReplicatedSize = new ArrayList<>();
+ this.objectUpdateIDs = new ArrayList<>();
+ }
+
+ public void add(String name, long size, long updateID) {
+ objectNames.add(name);
+ objectReplicatedSize.add(size);
+ objectUpdateIDs.add(updateID);
+ }
+
+ public void addAll(LimitedExpiredObjectList other) {
+ objectNames.addAll(other.objectNames);
+ objectReplicatedSize.addAll(other.objectReplicatedSize);
+ objectUpdateIDs.addAll(other.objectUpdateIDs);
+ }
+
+ public int size() {
+ return objectNames.size();
+ }
+
+ public List<String> nameSubList(int fromIndex, int toIndex) {
+ return objectNames.subList(fromIndex, toIndex);
+ }
+
+ public List<Long> updateIDSubList(int fromIndex, int toIndex) {
+ return objectUpdateIDs.subList(fromIndex, toIndex);
+ }
+
+ public List<Long> replicatedSizeSubList(int fromIndex, int toIndex) {
+ return objectReplicatedSize.subList(fromIndex, toIndex);
+ }
+
+ public Long getReplicatedSize(String keyName) {
+ for (int index = 0; index < objectNames.size(); index++) {
+ if (objectNames.get(index).equals(keyName)) {
+ return objectReplicatedSize.get(index);
+ }
+ }
+ return null;
+ }
+
+ public void clear() {
+ objectNames.clear();
+ objectUpdateIDs.clear();
+ objectReplicatedSize.clear();
+ }
+
+ public boolean isEmpty() {
+ return objectNames.isEmpty();
+ }
+
+ public boolean isFull() {
+ return objectNames.isFull();
+ }
+
+ public String getName(int index) {
+ return objectNames.get(index);
+ }
+
+ public long getUpdateID(int index) {
+ return objectUpdateIDs.get(index);
+ }
+
+ public long getReplicatedSize(int index) {
+ return objectReplicatedSize.get(index);
+ }
+ }
+
+ /**
+ * An in-memory list with a maximum size. This class is not thread safe.
+ */
+ public static class LimitedSizeList<T> {
+ private final List<T> internalList;
+ private final int maxSize;
+
+ public LimitedSizeList(int maxSize) {
+ this.maxSize = maxSize;
+ this.internalList = new ArrayList<>();
+ }
+
+ /**
+ * Add an element to the list. It blindly adds the element without check whether the list is full or not.
+ * Caller must check the size of the list through isFull() before calling this method.
+ */
+ public void add(T element) {
+ internalList.add(element);
+ }
+
+ public void addAll(LimitedSizeList<T> other) {
+ internalList.addAll(other.internalList);
+ }
+
+ public T get(int index) {
+ return internalList.get(index);
+ }
+
+ public int size() {
+ return internalList.size();
+ }
+
+ public List<T> subList(int fromIndex, int toIndex) {
+ return internalList.subList(fromIndex, toIndex);
+ }
+
+ public boolean isEmpty() {
+ return internalList.isEmpty();
+ }
+
+ public boolean isFull() {
+ boolean full = internalList.size() >= maxSize;
+ if (full) {
+ LOG.debug("LimitedSizeList has reached maximum size {}", maxSize);
+ }
+ return full;
+ }
+
+ public void clear() {
+ internalList.clear();
+ }
+ }
+
+ /**
+ * A list that tracks the total part count of multipart uploads.
+ * The list is considered "full" when the total part count reaches the limit.
+ * This is used because some MPUs might have 1 part while others might have 10,000 parts.
+ */
+ public static class PartCountLimitedList {
+ private final List<OmMultipartUpload> uploads;
+ private final int maxPartCount;
+ private int currentPartCount;
+
+ public PartCountLimitedList(int maxPartCount) {
+ this.maxPartCount = maxPartCount;
+ this.uploads = new ArrayList<>();
+ this.currentPartCount = 0;
+ }
+
+ /**
+ * Add a multipart upload with its part count.
+ * Caller should check isFull() before calling this method.
+ */
+ public void add(OmMultipartUpload upload, int partCount) {
+ uploads.add(upload);
+ currentPartCount += partCount;
+ }
+
+ public List<OmMultipartUpload> getUploads() {
+ return uploads;
+ }
+
+ public int size() {
+ return uploads.size();
+ }
+
+ public int getPartCount() {
+ return currentPartCount;
+ }
+
+ public boolean isEmpty() {
+ return uploads.isEmpty();
+ }
+
+ public boolean isFull() {
+ boolean full = currentPartCount >= maxPartCount;
+ if (full) {
+ LOG.debug("PartCountLimitedList has reached maximum part count {}", maxPartCount);
+ }
+ return full;
+ }
+
+ public void clear() {
+ uploads.clear();
+ currentPartCount = 0;
+ }
+ }
+
+ /**
+ * An in-memory class to hold the information required in directory recursive evaluation.
+ */
+ public static class PendingEvaluateDirectory {
+ private final OmDirectoryInfo directoryInfo;
+ private String dirTableKey;
+ private String dirPath;
+ private DirectoryList directoryList;
+ private boolean firstEvaluate;
+
+ public PendingEvaluateDirectory(OmDirectoryInfo dir, String dirTableKey, String dirPath, DirectoryList summary) {
+ this.directoryInfo = dir;
+ this.dirTableKey = dirTableKey;
+ this.dirPath = dirPath;
+ this.directoryList = summary;
+ this.firstEvaluate = true;
+ }
+
+ public String getDirTableKey() {
+ return dirTableKey;
+ }
+
+ public void setDirTableKey(String tableKey) {
+ dirTableKey = tableKey;
+ }
+
+ public String getDirPath() {
+ return dirPath;
+ }
+
+ public OmDirectoryInfo getDirectoryInfo() {
+ return directoryInfo;
+ }
+
+ public DirectoryList getSubDirSummary() {
+ return directoryList;
+ }
+
+ public void setDirectoryList(DirectoryList summary) {
+ directoryList = summary;
+ }
+
+ public boolean isFirstEvaluate() {
+ return firstEvaluate;
+ }
+
+ public void setFirstEvaluate(boolean firstEvaluate) {
+ this.firstEvaluate = firstEvaluate;
+ }
+ }
+
+ /**
+ * An in-memory class to hold a directory list.
+ */
+ public static class DirectoryList {
+ private final List<OmDirectoryInfo> subDirList;
+ private final List<String> subDirKeyList;
+ private final List<String> subDirKeyPathList;
+ private int subDirCount;
+ private boolean allResolvedPrefix = true;
+
+ public DirectoryList() {
+ this.subDirList = new ArrayList<>();
+ this.subDirKeyList = new ArrayList<>();
+ this.subDirKeyPathList = new ArrayList<>();
+ this.subDirCount = 0;
+ }
+
+ public int getSubDirCount() {
+ return subDirCount;
+ }
+
+ public List<OmDirectoryInfo> getSubDirList() {
+ return subDirList;
+ }
+
+ public List<String> getSubDirKeyList() {
+ return subDirKeyList;
+ }
+
+ public void addSubDir(String key, OmDirectoryInfo dir, String dirPath) {
+ subDirKeyList.add(key);
+ subDirList.add(dir);
+ subDirKeyPathList.add(dirPath);
+ subDirCount++;
+ }
+
+ public OmDirectoryInfo getLastSubDir() {
+ return subDirCount > 0 ? subDirList.get(subDirCount - 1) : null;
+ }
+
+ public String getLastSubDirKey() {
+ return subDirCount > 0 ? subDirKeyList.get(subDirCount - 1) : null;
+ }
+
+ public String getLastSubDirPath() {
+ return subDirCount > 0 ? subDirKeyPathList.get(subDirCount - 1) : null;
+ }
+
+ public boolean isAllResolvedPrefix() {
+ return allResolvedPrefix;
+ }
+
+ public void setNotAllResolvedPrefix() {
+ this.allResolvedPrefix = false;
+ }
+
+ public boolean isEmpty() {
+ return subDirCount == 0;
+ }
+
+ @Override
+ public String toString() {
+ return "DirectoryList { " +
+ "subDirList = " + subDirList +
+ ", subDirKeyList = " + subDirKeyList +
+ ", subDirKeyPathList = " + subDirKeyPathList +
+ ", subDirCount = " + subDirCount +
+ ", allResolvedPrefix = " + allResolvedPrefix +
+ '}';
+ }
+ }
+
+ /**
+ * An in-memory class to hold a rule list, together with the resolved prefix's directory list.
+ */
+ public static class RuleListWithDirectoryList {
+ private DirectoryList dirList;
+ private final List<OmLCRule> ruleList;
+ private String consolidatedPrefix;
+
+ public RuleListWithDirectoryList() {
+ this.ruleList = new ArrayList<>();
+ this.dirList = new DirectoryList();
+ }
+
+ public RuleListWithDirectoryList(List<OmLCRule> ruleList, DirectoryList dirList, String prefix) {
+ this.ruleList = ruleList;
+ this.dirList = dirList;
+ this.consolidatedPrefix = prefix;
+ }
+
+ public List<OmLCRule> getRuleList() {
+ return ruleList;
+ }
+
+ public DirectoryList getDirList() {
+ return dirList;
+ }
+
+ public void addRule(OmLCRule rule) {
+ this.ruleList.add(rule);
+ }
+
+ public void setDirList(DirectoryList dirList) {
+ this.dirList = dirList;
+ }
+
+ public String getConsolidatedPrefix() {
+ return consolidatedPrefix;
+ }
+
+ public void setConsolidatedPrefix(String consolidatedPrefix) {
+ this.consolidatedPrefix = consolidatedPrefix;
+ }
+
+ public boolean isEmpty() {
+ return dirList.isEmpty() && ruleList.isEmpty() && consolidatedPrefix == null;
+ }
+
+ @Override
+ public String toString() {
+ return "RuleListWithDirectoryList { " +
+ "dirList = " + dirList +
+ ", ruleList = " + ruleList +
+ ", consolidatedPrefix = '" + consolidatedPrefix + '\'' +
+ '}';
+ }
+ }
+
+ /**
+ * Orders of RuleListWithDirectoryList.
+ */
+ public static class RuleListWithDirectoryListOrder implements Comparator<RuleListWithDirectoryList> {
+
+ public static final Comparator<RuleListWithDirectoryList> INSTANCE =
+ new RuleListWithDirectoryListOrder();
+
+ @Override
+ public int compare(RuleListWithDirectoryList o1, RuleListWithDirectoryList o2) {
+ List<OmDirectoryInfo> dirList1 = o1.getDirList().getSubDirList();
+ List<OmDirectoryInfo> dirList2 = o2.getDirList().getSubDirList();
+
+ // find their shared parent directory
+ OmDirectoryInfo parent = null;
+ for (int i = dirList1.size() - 1; i >= 0; i--) {
+ long objID = dirList1.get(i).getObjectID();
+ for (int j = dirList2.size() - 1; j >= 0; j--) {
+ if (dirList2.get(j).getObjectID() == objID) {
+ parent = dirList2.get(j);
+ break;
+ }
+ }
+ if (parent != null) {
+ break;
+ }
+ }
+ if (parent == null) {
+ // e.g, dir1 and dir2, dir2 should ahead of dir2, given the depth-first and dir's RocksDB key order
+ return dirList2.get(0).getName().compareTo(dirList1.get(0).getName());
+ } else {
+ long parentID = parent.getObjectID();
+ OmDirectoryInfo dir1 = dirList1.stream().filter(dir -> dir.getParentObjectID() == parentID)
+ .collect(Collectors.toList()).get(0);
+ OmDirectoryInfo dir2 = dirList2.stream().filter(dir -> dir.getParentObjectID() == parentID)
+ .collect(Collectors.toList()).get(0);
+ return dir2.getName().compareTo(dir1.getName());
+ }
+ }
+ }
+
+ /**
+ * An in-memory stack with a maximum size. This class is not thread safe.
+ */
+ public static class LimitedSizeStack {
+ private final Deque<PendingEvaluateDirectory> stack;
+ private final long maxSize;
+
+ public LimitedSizeStack(long maxSize) {
+ this.maxSize = maxSize;
+ this.stack = new ArrayDeque<>();
+ }
+
+ public boolean isEmpty() {
+ return stack.isEmpty();
+ }
+
+ public void push(PendingEvaluateDirectory e) throws CapacityFullException {
+ if (stack.size() >= maxSize) {
+ throw new CapacityFullException("LimitedSizeStack has reached maximum size " + maxSize);
+ }
+ stack.push(e);
+ }
+
+ public PendingEvaluateDirectory pop() {
+ return stack.pop();
+ }
+ }
+
+ /**
+ * An exception which indicates the collection is full.
+ */
+ public static class CapacityFullException extends Exception {
+ public CapacityFullException(String message) {
+ super(message);
+ }
+ }
+
+ public static void setTest(boolean test) {
+ KeyLifecycleService.test = test;
+ }
+
+ public static void reSetConsolidatedRuleList() {
+ consolidatedRuleList = null;
+ }
+
+ public static List<RuleListWithDirectoryList> getConsolidatedRuleList() {
+ return consolidatedRuleList;
+ }
+ }
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
index 346b6524f60,189bb3f3e1d..fbda6b78c5d
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
@@@ -52,18 -44,15 +52,20 @@@ public enum OMLayoutFeature implements
QUOTA(6, "Ozone quota re-calculate"),
HBASE_SUPPORT(7, "Full support of hsync, lease recovery and listOpenFiles APIs for HBase"),
DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric key"),
- SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot");
+ SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot"),
+ S3_LIFECYCLE_SUPPORT(10, "S3 bucket lifecycle configuration support"),
+ MPU_PARTS_TABLE_SPLIT(11, "Split multipart table into separate table for parts and key");
+ // ALL NEW VERSIONS SHOULD NOW BE ADDED TO OzoneManagerVersion
+
/////////////////////////////// /////////////////////////////
- private int layoutVersion;
- private String description;
- private OmUpgradeAction action;
+ private static final SortedMap<Integer, OMLayoutFeature> BY_VALUE =
+ Arrays.stream(values())
+ .collect(toMap(OMLayoutFeature::serialize, identity(), (v1, v2) -> v1, TreeMap::new));
+
+ private final int layoutVersion;
+ private final String description;
OMLayoutFeature(final int layoutVersion, String description) {
this.layoutVersion = layoutVersion;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 9b90eb81a5d,b585fa234c8..c2b8c33accb
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@@ -1413,33 -1415,31 +1430,50 @@@ public class OzoneManagerRequestHandle
.build();
}
- private PrepareStatusResponse getPrepareStatus() {
- OzoneManagerPrepareState.State prepareState =
- impl.getPrepareState().getState();
+ private PrepareStatusResponse getPrepareStatus() throws IOException {
+ impl.checkAdminUserPrivilege("get prepare status");
+
+ // Prepare is no longer used, always return PREPARE_COMPLETED for backward compatibility
return PrepareStatusResponse.newBuilder()
- .setStatus(prepareState.getStatus())
- .setCurrentTxnIndex(prepareState.getIndex()).build();
+ .setStatus(PrepareStatusResponse.PrepareStatus.PREPARE_COMPLETED)
+ .setCurrentTxnIndex(0).build();
+ }
+
+ private PrepareResponse prepare() throws IOException {
+ // Prepare functionality is no longer supported. An empty successful transaction is returned for compatibility
+ // with older clients and upgrade processes.
+ impl.checkAdminUserPrivilege("prepare for upgrade");
+
+ return PrepareResponse.newBuilder()
+ .setTxnID(0) // Dummy transaction ID
+ .build();
+ }
+
+ private CancelPrepareResponse cancelPrepare() throws IOException {
+ // Cancel Prepare functionality is no longer supported. An empty successful transaction is returned for
+ // compatibility with older clients and upgrade processes.
+ impl.checkAdminUserPrivilege("cancel prepare");
+
+ return CancelPrepareResponse.newBuilder().build();
}
+ private GetLifecycleConfigurationResponse infoLifecycleConfiguration(
+ GetLifecycleConfigurationRequest request) throws IOException {
+
+ GetLifecycleConfigurationResponse.Builder resp =
+ GetLifecycleConfigurationResponse.newBuilder();
+
+ String volume = request.getVolumeName();
+ String bucket = request.getBucketName();
+
+ OmLifecycleConfiguration omLifecycleConfiguration =
+ impl.getLifecycleConfiguration(volume, bucket);
+
+ resp.setLifecycleConfiguration(omLifecycleConfiguration.getProtobuf());
+
+ return resp.build();
+ }
+
private GetS3VolumeContextResponse getS3VolumeContext()
throws IOException {
return impl.getS3VolumeContext().getProtobuf();
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationDeleteRequest.java
index 00000000000,ed7a9e146a0..386d6d24ec8
mode 000000,100644..100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationDeleteRequest.java
@@@ -1,0 -1,300 +1,296 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.ozone.om.request.lifecycle;
+
+ import static org.apache.hadoop.ozone.om.request.validation.ValidationContext.of;
+ import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertNotEquals;
+ import static org.junit.jupiter.api.Assertions.assertNotNull;
+ import static org.junit.jupiter.api.Assertions.assertNull;
+ import static org.junit.jupiter.api.Assertions.assertThrows;
+ import static org.mockito.Mockito.any;
+ import static org.mockito.Mockito.anyString;
+ import static org.mockito.Mockito.doNothing;
+ import static org.mockito.Mockito.eq;
+ import static org.mockito.Mockito.isNull;
+ import static org.mockito.Mockito.mock;
+ import static org.mockito.Mockito.spy;
+ import static org.mockito.Mockito.verify;
+ import static org.mockito.Mockito.when;
+
+ import java.io.IOException;
+ import java.util.UUID;
+ import org.apache.commons.lang3.tuple.Pair;
+ import org.apache.hadoop.ozone.om.ResolvedBucket;
+ import org.apache.hadoop.ozone.om.exceptions.OMException;
+ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+ import org.apache.hadoop.ozone.om.request.OMClientRequest;
+ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+ import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
+ import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
++import org.apache.hadoop.ozone.om.upgrade.OMVersionManager;
++import org.apache.hadoop.ozone.om.upgrade.OMVersionManagerTestUtils;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+ import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+ import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.upgrade.LayoutVersionManager;
+ import org.apache.hadoop.security.UserGroupInformation;
+ import org.junit.jupiter.api.Test;
+
+ /**
+ * Test class for delete Lifecycle configuration request.
+ */
+ public class TestOMLifecycleConfigurationDeleteRequest extends
+ TestOMLifecycleConfigurationRequest {
+ @Test
+ public void testPreExecute() throws Exception {
+ OMRequest omRequest = createDeleteLifecycleConfigurationRequest(
+ UUID.randomUUID().toString(), UUID.randomUUID().toString());
+
+ OMLifecycleConfigurationDeleteRequest request =
+ new OMLifecycleConfigurationDeleteRequest(omRequest);
+
+ // As user info gets added.
+ assertNotEquals(omRequest, request.preExecute(ozoneManager));
+ }
+
+ @Test
+ public void testPreExecuteWithLinkedBucket() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String resolvedBucketName = bucketName + "-resolved";
+ String resolvedVolumeName = volumeName + "-resolved";
+ // Mock the bucket link resolution
+ when(ozoneManager.resolveBucketLink(any(Pair.class), any(OMClientRequest.class)))
+ .thenAnswer(i -> new ResolvedBucket(i.getArgument(0),
+ Pair.of(resolvedVolumeName, resolvedBucketName),
+ "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED));
+
+ OMRequest omRequest = createDeleteLifecycleConfigurationRequest(volumeName, bucketName);
+ OMLifecycleConfigurationDeleteRequest request =
+ new OMLifecycleConfigurationDeleteRequest(omRequest);
+ OMRequest modifiedRequest = request.preExecute(ozoneManager);
+
+ // Verify that the resolved volume and bucket names are used
+ assertEquals(resolvedVolumeName,
+ modifiedRequest.getDeleteLifecycleConfigurationRequest().getVolumeName());
+ assertEquals(resolvedBucketName,
+ modifiedRequest.getDeleteLifecycleConfigurationRequest().getBucketName());
+ }
+
+ @Test
+ public void testValidateAndUpdateCache() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ // Create Volume and bucket entries in DB.
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager);
+
+ addLifecycleConfigurationToDB(volumeName, bucketName, "ownername");
+ assertNotNull(omMetadataManager.getLifecycleConfigurationTable()
+ .get(omMetadataManager.getBucketKey(volumeName, bucketName)));
+
+ OMRequest omRequest =
+ createDeleteLifecycleConfigurationRequest(volumeName, bucketName);
+
+ OMLifecycleConfigurationDeleteRequest deleteRequest =
+ new OMLifecycleConfigurationDeleteRequest(omRequest);
+
+ deleteRequest.validateAndUpdateCache(ozoneManager, 1L);
+
+ assertNull(omMetadataManager.getLifecycleConfigurationTable().get(
+ omMetadataManager.getBucketKey(volumeName, bucketName)));
+ }
+
+ @Test
+ public void testValidateAndUpdateCacheFailure() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ // Create Volume and bucket entries in DB.
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager);
+
+ OMRequest omRequest =
+ createDeleteLifecycleConfigurationRequest(volumeName, bucketName);
+
+ OMLifecycleConfigurationDeleteRequest deleteRequest =
+ new OMLifecycleConfigurationDeleteRequest(omRequest);
+
+ OMClientResponse omClientResponse = deleteRequest.validateAndUpdateCache(
+ ozoneManager, 1L);
+
+ OMResponse omResponse = omClientResponse.getOMResponse();
+ assertEquals(
+ OzoneManagerProtocolProtos.Status.LIFECYCLE_CONFIGURATION_NOT_FOUND,
+ omResponse.getStatus());
+ }
+
+ private void addLifecycleConfigurationToDB(String volumeName,
+ String bucketName, String ownerName) throws IOException {
+ OMRequest originalRequest = setLifecycleConfigurationRequest(volumeName,
+ bucketName, ownerName);
+
+ OMLifecycleConfigurationSetRequest request =
+ new OMLifecycleConfigurationSetRequest(originalRequest);
+
+ OMRequest modifiedRequest = request.preExecute(ozoneManager);
+
+ String lifecycleKey = omMetadataManager.getBucketKey(volumeName,
+ bucketName);
+
+ assertNull(omMetadataManager.getLifecycleConfigurationTable().get(
+ lifecycleKey));
+
+ request = new OMLifecycleConfigurationSetRequest(modifiedRequest);
+ long txLogIndex = 1;
+
+ request.validateAndUpdateCache(ozoneManager, txLogIndex);
+ }
+
+ @Test
+ public void testDisallowDeleteLifecycleConfigurationBeforeFinalization() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
- LayoutVersionManager versionManager = mock(LayoutVersionManager.class);
- when(versionManager.isAllowed(OMLayoutFeature.S3_LIFECYCLE_SUPPORT)).thenReturn(false);
-
++ OMVersionManager versionManager = OMVersionManagerTestUtils.mockPreFinalizedOmVersionManager();
+ ValidationContext ctx = of(versionManager, omMetadataManager);
+ OMRequest request = createDeleteLifecycleConfigurationRequest(volumeName, bucketName);
+
+ OMException ex = assertThrows(OMException.class, () ->
+ OMLifecycleConfigurationDeleteRequest
+ .disallowDeleteLifecycleConfigurationBeforeFinalization(request, ctx));
+
+ assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION,
+ ex.getResult());
+ }
+
+ @Test
+ public void testPreExecuteNonNativeAuthorizerChecksWriteAcl() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ IAccessAuthorizer authorizer = mock(IAccessAuthorizer.class);
+ when(authorizer.isNative()).thenReturn(false);
+ when(ozoneManager.getAccessAuthorizer()).thenReturn(authorizer);
+
+ OMRequest omRequest =
+ createDeleteLifecycleConfigurationRequest(volumeName, bucketName);
+ OMLifecycleConfigurationDeleteRequest request =
+ spy(new OMLifecycleConfigurationDeleteRequest(omRequest));
+ // Stub the ACL check so the branch can be asserted without a real authorizer.
+ doNothing().when(request).checkAcls(eq(ozoneManager), eq(OzoneObj.ResourceType.BUCKET),
+ eq(OzoneObj.StoreType.OZONE), eq(IAccessAuthorizer.ACLType.WRITE),
+ eq(volumeName), eq(bucketName), isNull());
+
+ request.preExecute(ozoneManager);
+
+ verify(request).checkAcls(eq(ozoneManager), eq(OzoneObj.ResourceType.BUCKET),
+ eq(OzoneObj.StoreType.OZONE), eq(IAccessAuthorizer.ACLType.WRITE),
+ eq(volumeName), eq(bucketName), isNull());
+ }
+
+ @Test
+ public void testPreExecuteNativeAuthorizerDeniesNonAdminNonOwner() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ IAccessAuthorizer authorizer = mock(IAccessAuthorizer.class);
+ when(authorizer.isNative()).thenReturn(true);
+ when(ozoneManager.getAccessAuthorizer()).thenReturn(authorizer);
+ when(ozoneManager.getBucketOwner(eq(volumeName), eq(bucketName),
+ any(IAccessAuthorizer.ACLType.class), any(OzoneObj.ResourceType.class)))
+ .thenReturn("bucketOwner");
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false);
+ when(ozoneManager.isOwner(any(UserGroupInformation.class), anyString())).thenReturn(false);
+
+ OMRequest omRequest =
+ createDeleteLifecycleConfigurationRequest(volumeName, bucketName);
+ OMLifecycleConfigurationDeleteRequest request =
+ new OMLifecycleConfigurationDeleteRequest(omRequest);
+ request.setUGI(UserGroupInformation.createRemoteUser("regularUser"));
+
+ OMException ex = assertThrows(OMException.class,
+ () -> request.preExecute(ozoneManager));
+ assertEquals(OMException.ResultCodes.PERMISSION_DENIED, ex.getResult());
+ }
+
+ @Test
+ public void testPreExecuteNativeAuthorizerAllowsAdmin() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ IAccessAuthorizer authorizer = mock(IAccessAuthorizer.class);
+ when(authorizer.isNative()).thenReturn(true);
+ when(ozoneManager.getAccessAuthorizer()).thenReturn(authorizer);
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(true);
+
+ OMRequest omRequest =
+ createDeleteLifecycleConfigurationRequest(volumeName, bucketName);
+ OMLifecycleConfigurationDeleteRequest request =
+ new OMLifecycleConfigurationDeleteRequest(omRequest);
+ request.setUGI(UserGroupInformation.createRemoteUser("adminUser"));
+
+ assertNotNull(request.preExecute(ozoneManager));
+ }
+
+ @Test
+ public void testPreExecuteNativeAuthorizerAllowsOwner() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ IAccessAuthorizer authorizer = mock(IAccessAuthorizer.class);
+ when(authorizer.isNative()).thenReturn(true);
+ when(ozoneManager.getAccessAuthorizer()).thenReturn(authorizer);
+ when(ozoneManager.getBucketOwner(eq(volumeName), eq(bucketName),
+ any(IAccessAuthorizer.ACLType.class), any(OzoneObj.ResourceType.class)))
+ .thenReturn("bucketOwner");
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false);
+ when(ozoneManager.isOwner(any(UserGroupInformation.class), eq("bucketOwner")))
+ .thenReturn(true);
+
+ OMRequest omRequest =
+ createDeleteLifecycleConfigurationRequest(volumeName, bucketName);
+ OMLifecycleConfigurationDeleteRequest request =
+ new OMLifecycleConfigurationDeleteRequest(omRequest);
+ request.setUGI(UserGroupInformation.createRemoteUser("ownerUser"));
+
+ assertNotNull(request.preExecute(ozoneManager));
+ }
+
+ @Test
+ public void testAllowDeleteLifecycleConfigurationAfterFinalization() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
- LayoutVersionManager versionManager = mock(LayoutVersionManager.class);
- when(versionManager.isAllowed(OMLayoutFeature.S3_LIFECYCLE_SUPPORT)).thenReturn(true);
-
++ OMVersionManager versionManager = OMVersionManagerTestUtils.mockFinalizedOmVersionManager();
+ ValidationContext ctx = of(versionManager, omMetadataManager);
+ OMRequest request = createDeleteLifecycleConfigurationRequest(volumeName, bucketName);
+
+ OMRequest result = OMLifecycleConfigurationDeleteRequest
+ .disallowDeleteLifecycleConfigurationBeforeFinalization(request, ctx);
+
+ assertEquals(request, result);
+ }
+ }
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
index 00000000000,ddc581c5029..70c622d373f
mode 000000,100644..100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
@@@ -1,0 -1,153 +1,153 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.ozone.om.request.lifecycle;
+
+ import static org.mockito.ArgumentMatchers.any;
+ import static org.mockito.Mockito.mock;
+ import static org.mockito.Mockito.when;
+
+ import java.io.File;
+ import java.util.UUID;
+ import org.apache.commons.lang3.RandomStringUtils;
+ import org.apache.commons.lang3.tuple.Pair;
+ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+ import org.apache.hadoop.ozone.audit.AuditLogger;
+ import org.apache.hadoop.ozone.audit.AuditMessage;
+ import org.apache.hadoop.ozone.om.OMConfigKeys;
+ import org.apache.hadoop.ozone.om.OMMetadataManager;
+ import org.apache.hadoop.ozone.om.OMMetrics;
+ import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+ import org.apache.hadoop.ozone.om.OzoneManager;
+ import org.apache.hadoop.ozone.om.ResolvedBucket;
+ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+ import org.apache.hadoop.ozone.om.request.OMClientRequest;
+ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
-import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
++import org.apache.hadoop.ozone.om.upgrade.OMVersionManager;
++import org.apache.hadoop.ozone.om.upgrade.OMVersionManagerTestUtils;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketLayoutProto;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteLifecycleConfigurationRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleConfiguration;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleExpiration;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleFilter;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRule;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetLifecycleConfigurationRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+ import org.junit.jupiter.api.AfterEach;
+ import org.junit.jupiter.api.BeforeEach;
+ import org.junit.jupiter.api.io.TempDir;
+ import org.mockito.Mockito;
+
+ /**
+ * Base test class for Lifecycle configuration request.
+ */
+ @SuppressWarnings("visibilitymodifier")
+ public class TestOMLifecycleConfigurationRequest {
+
+ @TempDir
+ private File tempDir;
+
+ protected OzoneManager ozoneManager;
+ protected OMMetrics omMetrics;
+ protected OMMetadataManager omMetadataManager;
+ protected AuditLogger auditLogger;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ ozoneManager = mock(OzoneManager.class);
+ OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+ omMetrics = OMMetrics.create(ozoneConfiguration);
+ ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, tempDir.getAbsolutePath());
+ omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager);
+ when(ozoneManager.getMetrics()).thenReturn(omMetrics);
+ when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
+ when(ozoneManager.getMaxUserVolumeCount()).thenReturn(10L);
+ when(ozoneManager.resolveBucketLink(any(Pair.class), any(OMClientRequest.class)))
+ .thenAnswer(i -> new ResolvedBucket(i.getArgument(0),
+ i.getArgument(0), "dummyBucketOwner", BucketLayout.OBJECT_STORE));
+ when(ozoneManager.isStrictS3()).thenReturn(true);
- OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
- when(lvm.getMetadataLayoutVersion()).thenReturn(0);
- when(ozoneManager.getVersionManager()).thenReturn(lvm);
++ OMVersionManager versionManager = OMVersionManagerTestUtils.mockPreFinalizedOmVersionManager();
++ when(ozoneManager.getVersionManager()).thenReturn(versionManager);
+ auditLogger = mock(AuditLogger.class);
+ when(ozoneManager.getAuditLogger()).thenReturn(auditLogger);
+ Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
+ }
+
+ @AfterEach
+ public void stop() {
+ omMetrics.unRegister();
+ Mockito.framework().clearInlineMocks();
+ }
+
+ public OMRequest createDeleteLifecycleConfigurationRequest(
+ String volumeName, String bucketName) {
+ return OMRequest.newBuilder().setDeleteLifecycleConfigurationRequest(
+ DeleteLifecycleConfigurationRequest.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName))
+ .setCmdType(Type.DeleteLifecycleConfiguration)
+ .setClientId(UUID.randomUUID().toString()).build();
+ }
+
+ public static void addVolumeAndBucketToTable(String volumeName,
+ String bucketName, String ownerName, OMMetadataManager omMetadataManager)
+ throws Exception {
+ OMRequestTestUtils.addVolumeToDB(volumeName, ownerName, omMetadataManager);
+ OMRequestTestUtils.addBucketToDB(volumeName, bucketName, omMetadataManager);
+ }
+
+ public OMRequest setLifecycleConfigurationRequest(String volumeName,
+ String bucketName, String ownerName) {
+ return setLifecycleConfigurationRequest(volumeName, bucketName,
+ ownerName, true);
+ }
+
+ public OMRequest setLifecycleConfigurationRequest(String volumeName,
+ String bucketName, String ownerName, boolean addRules) {
+ String prefix = "prefix/";
+ LifecycleConfiguration.Builder builder = LifecycleConfiguration.newBuilder()
+ .setBucketLayout(BucketLayoutProto.OBJECT_STORE)
+ .setCreationTime(System.currentTimeMillis())
+ .setVolume(volumeName)
+ .setBucket(bucketName);
+
+ if (addRules) {
+ builder.addRules(LifecycleRule.newBuilder()
+ .setId(RandomStringUtils.randomAlphabetic(32))
+ .setEnabled(true)
+ .addAction(LifecycleAction.newBuilder()
+ .setExpiration(LifecycleExpiration.newBuilder().setDays(3).build()))
+ .setFilter(LifecycleFilter.newBuilder().setPrefix(prefix))
+ );
+ }
+
+ LifecycleConfiguration lcc = builder.build();
+
+ SetLifecycleConfigurationRequest setLifecycleConfigurationRequest =
+ SetLifecycleConfigurationRequest.newBuilder()
+ .setLifecycleConfiguration(lcc)
+ .build();
+
+ return OMRequest.newBuilder().setSetLifecycleConfigurationRequest(
+ setLifecycleConfigurationRequest)
+ .setCmdType(Type.SetLifecycleConfiguration)
+ .setClientId(UUID.randomUUID().toString())
+ .build();
+ }
+ }
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
index 00000000000,721a012f5a7..df3758bd46d
mode 000000,100644..100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
@@@ -1,0 -1,390 +1,386 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.ozone.om.request.lifecycle;
+
+ import static org.apache.hadoop.ozone.om.request.validation.ValidationContext.of;
+ import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertNotEquals;
+ import static org.junit.jupiter.api.Assertions.assertNotNull;
+ import static org.junit.jupiter.api.Assertions.assertNull;
+ import static org.junit.jupiter.api.Assertions.assertThrows;
+ import static org.mockito.Mockito.any;
+ import static org.mockito.Mockito.anyString;
+ import static org.mockito.Mockito.doNothing;
+ import static org.mockito.Mockito.eq;
+ import static org.mockito.Mockito.isNull;
+ import static org.mockito.Mockito.mock;
+ import static org.mockito.Mockito.spy;
+ import static org.mockito.Mockito.verify;
+ import static org.mockito.Mockito.when;
+
+ import java.util.UUID;
+ import org.apache.commons.lang3.tuple.Pair;
+ import org.apache.hadoop.ozone.om.ResolvedBucket;
+ import org.apache.hadoop.ozone.om.exceptions.OMException;
+ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+ import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
+ import org.apache.hadoop.ozone.om.request.OMClientRequest;
+ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+ import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
+ import org.apache.hadoop.ozone.om.response.OMClientResponse;
+ import org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleConfigurationSetResponse;
-import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
++import org.apache.hadoop.ozone.om.upgrade.OMVersionManager;
++import org.apache.hadoop.ozone.om.upgrade.OMVersionManagerTestUtils;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleConfiguration;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+ import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+ import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.upgrade.LayoutVersionManager;
+ import org.apache.hadoop.security.UserGroupInformation;
+ import org.junit.jupiter.api.Test;
+
+ /**
+ * Test class for create Lifecycle configuration request.
+ */
+ public class TestOMLifecycleConfigurationSetRequest extends
+ TestOMLifecycleConfigurationRequest {
+ @Test
+ public void testPreExecute() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ doPreExecute(volumeName, bucketName);
+
+ // Volume name and bucket name length should be greater than OZONE_MIN_BUCKET_NAME_LENGTH
+ assertThrows(OMException.class, () -> doPreExecute("v1", "bucket1"));
+ assertThrows(OMException.class, () -> doPreExecute("volume1", "b1"));
+ }
+
+ @Test
+ public void testPreExecuteWithLinkedBucket() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String resolvedBucketName = bucketName + "-resolved";
+ String resolvedVolumeName = volumeName + "-resolved";
+ // Mock the bucket link resolution
+ when(ozoneManager.resolveBucketLink(any(Pair.class), any(OMClientRequest.class)))
+ .thenAnswer(i -> new ResolvedBucket(i.getArgument(0),
+ Pair.of(resolvedVolumeName, resolvedBucketName),
+ "owner", BucketLayout.OBJECT_STORE));
+ OMRequest originalRequest = setLifecycleConfigurationRequest(volumeName, bucketName, "ownername");
+ OMLifecycleConfigurationSetRequest request = new OMLifecycleConfigurationSetRequest(originalRequest);
+ OMRequest modifiedRequest = request.preExecute(ozoneManager);
+
+ // Verify that the resolved volume and bucket names are used in the lifecycle configuration
+ LifecycleConfiguration lifecycleConfig =
+ modifiedRequest.getSetLifecycleConfigurationRequest().getLifecycleConfiguration();
+ assertEquals(resolvedVolumeName, lifecycleConfig.getVolume());
+ assertEquals(resolvedBucketName, lifecycleConfig.getBucket());
+ }
+
+ @Test
+ public void testValidateAndUpdateCacheSuccess() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String ownerName = "ownerName";
+
+ addVolumeAndBucketToTable(volumeName, bucketName, ownerName,
+ omMetadataManager);
+
+ OMRequest originalRequest = setLifecycleConfigurationRequest(volumeName,
+ bucketName, ownerName);
+
+ OMLifecycleConfigurationSetRequest request =
+ new OMLifecycleConfigurationSetRequest(originalRequest);
+
+ OMRequest modifiedRequest = request.preExecute(ozoneManager);
+
+ String lifecycleKey = omMetadataManager.getBucketKey(volumeName,
+ bucketName);
+
+ assertNull(omMetadataManager.getLifecycleConfigurationTable().get(
+ lifecycleKey));
+
+ request = new OMLifecycleConfigurationSetRequest(modifiedRequest);
+ long txLogIndex = 2;
+
+ OMClientResponse omClientResponse = request.validateAndUpdateCache(ozoneManager, txLogIndex);
+ OMResponse omResponse = omClientResponse.getOMResponse();
+ OMLifecycleConfigurationSetResponse response = (OMLifecycleConfigurationSetResponse) omClientResponse;
+ assertNotNull(response.getOmLifecycleConfiguration().getBucketObjectID());
+ assertNotNull(omResponse.getSetLifecycleConfigurationResponse());
+ assertEquals(OzoneManagerProtocolProtos.Status.OK,
+ omResponse.getStatus());
+ assertEquals(Type.SetLifecycleConfiguration,
+ omResponse.getCmdType());
+
+ LifecycleConfiguration lifecycleConfigurationRequestProto =
+ request.getOmRequest()
+ .getSetLifecycleConfigurationRequest()
+ .getLifecycleConfiguration();
+
+ OmLifecycleConfiguration lifecycleConfigurationRequest =
+ OmLifecycleConfiguration.getFromProtobuf(
+ lifecycleConfigurationRequestProto);
+
+ OmLifecycleConfiguration lifecycleConfiguration = omMetadataManager
+ .getLifecycleConfigurationTable().get(lifecycleKey);
+
+ assertNotNull(lifecycleConfiguration);
+ assertEquals(lifecycleConfigurationRequest.getVolume(),
+ lifecycleConfiguration.getVolume());
+ assertEquals(lifecycleConfigurationRequest.getBucket(),
+ lifecycleConfiguration.getBucket());
+ assertEquals(lifecycleConfigurationRequest.getBucket(),
+ lifecycleConfiguration.getBucket());
+ assertEquals(lifecycleConfigurationRequest.getCreationTime(),
+ lifecycleConfiguration.getCreationTime());
+ }
+
+ @Test
+ public void testValidateAndUpdateNoBucket() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String ownerName = "ownerName";
+
+ OMRequestTestUtils.addVolumeToDB(volumeName, ownerName, omMetadataManager);
+
+ OMRequest originalRequest = setLifecycleConfigurationRequest(volumeName,
+ bucketName, ownerName);
+
+ OMLifecycleConfigurationSetRequest request =
+ new OMLifecycleConfigurationSetRequest(originalRequest);
+
+ OMRequest modifiedRequest = request.preExecute(ozoneManager);
+
+ String lifecycleKey = omMetadataManager.getBucketKey(volumeName,
+ bucketName);
+
+ assertNull(omMetadataManager.getLifecycleConfigurationTable().get(
+ lifecycleKey));
+
+ request = new OMLifecycleConfigurationSetRequest(modifiedRequest);
+ long txLogIndex = 2;
+
+ OMClientResponse omClientResponse = request.validateAndUpdateCache(ozoneManager, txLogIndex);
+ OMResponse omResponse = omClientResponse.getOMResponse();
+
+ assertEquals(OzoneManagerProtocolProtos.Status.BUCKET_NOT_FOUND,
+ omResponse.getStatus());
+ }
+
+ @Test
+ public void testValidateAndUpdateInvalidLCC() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+ String ownerName = "ownerName";
+
+ OMRequestTestUtils.addVolumeToDB(volumeName, ownerName, omMetadataManager);
+ OMRequestTestUtils.addBucketToDB(volumeName, bucketName, omMetadataManager);
+
+ OMRequest originalRequest = setLifecycleConfigurationRequest(volumeName,
+ bucketName, ownerName, false);
+
+ OMLifecycleConfigurationSetRequest request =
+ new OMLifecycleConfigurationSetRequest(originalRequest);
+
+ OMRequest modifiedRequest = request.preExecute(ozoneManager);
+
+ String lifecycleKey = omMetadataManager.getBucketKey(volumeName,
+ bucketName);
+
+ assertNull(omMetadataManager.getLifecycleConfigurationTable().get(
+ lifecycleKey));
+
+ request = new OMLifecycleConfigurationSetRequest(modifiedRequest);
+ long txLogIndex = 2;
+
+ OMClientResponse omClientResponse = request.validateAndUpdateCache(ozoneManager, txLogIndex);
+ OMResponse omResponse = omClientResponse.getOMResponse();
+
+ assertEquals(OzoneManagerProtocolProtos.Status.INVALID_REQUEST,
+ omResponse.getStatus());
+ }
+
+ private void doPreExecute(String volumeName, String bucketName)
+ throws Exception {
+
+ OMRequest originalRequest = setLifecycleConfigurationRequest(volumeName,
+ bucketName, "ownername");
+
+ OMLifecycleConfigurationSetRequest request =
+ new OMLifecycleConfigurationSetRequest(originalRequest);
+
+ // sleep to make sure two requests' timestamp will be different.
+ Thread.sleep(1);
+ OMRequest modifiedRequest = request.preExecute(ozoneManager);
+ verifyRequest(modifiedRequest, originalRequest);
+ }
+
+ /**
+ * Verify modifiedOmRequest and originalRequest.
+ * @param modifiedRequest
+ * @param originalRequest
+ */
+ private void verifyRequest(OMRequest modifiedRequest,
+ OMRequest originalRequest) {
+
+ LifecycleConfiguration original =
+ originalRequest.getSetLifecycleConfigurationRequest()
+ .getLifecycleConfiguration();
+
+ LifecycleConfiguration updated =
+ modifiedRequest.getSetLifecycleConfigurationRequest()
+ .getLifecycleConfiguration();
+
+ assertEquals(original.getVolume(), updated.getVolume());
+ assertEquals(original.getBucket(), updated.getBucket());
+ assertNotEquals(original.getCreationTime(), updated.getCreationTime());
+ assertEquals(original.getRulesList(), updated.getRulesList());
+ }
+
+ @Test
+ public void testPreExecuteNonNativeAuthorizerChecksWriteAcl() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ IAccessAuthorizer authorizer = mock(IAccessAuthorizer.class);
+ when(authorizer.isNative()).thenReturn(false);
+ when(ozoneManager.getAccessAuthorizer()).thenReturn(authorizer);
+
+ OMRequest omRequest =
+ setLifecycleConfigurationRequest(volumeName, bucketName, "ownerName");
+ OMLifecycleConfigurationSetRequest request =
+ spy(new OMLifecycleConfigurationSetRequest(omRequest));
+ // Stub the ACL check so the branch can be asserted without a real authorizer.
+ doNothing().when(request).checkAcls(eq(ozoneManager), eq(OzoneObj.ResourceType.BUCKET),
+ eq(OzoneObj.StoreType.OZONE), eq(IAccessAuthorizer.ACLType.WRITE),
+ eq(volumeName), eq(bucketName), isNull());
+
+ request.preExecute(ozoneManager);
+
+ verify(request).checkAcls(eq(ozoneManager), eq(OzoneObj.ResourceType.BUCKET),
+ eq(OzoneObj.StoreType.OZONE), eq(IAccessAuthorizer.ACLType.WRITE),
+ eq(volumeName), eq(bucketName), isNull());
+ }
+
+ @Test
+ public void testPreExecuteNativeAuthorizerDeniesNonAdminNonOwner() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ IAccessAuthorizer authorizer = mock(IAccessAuthorizer.class);
+ when(authorizer.isNative()).thenReturn(true);
+ when(ozoneManager.getAccessAuthorizer()).thenReturn(authorizer);
+ when(ozoneManager.getBucketOwner(eq(volumeName), eq(bucketName),
+ any(IAccessAuthorizer.ACLType.class), any(OzoneObj.ResourceType.class)))
+ .thenReturn("bucketOwner");
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false);
+ when(ozoneManager.isOwner(any(UserGroupInformation.class), anyString())).thenReturn(false);
+
+ OMRequest omRequest =
+ setLifecycleConfigurationRequest(volumeName, bucketName, "ownerName");
+ OMLifecycleConfigurationSetRequest request =
+ new OMLifecycleConfigurationSetRequest(omRequest);
+ request.setUGI(UserGroupInformation.createRemoteUser("regularUser"));
+
+ OMException ex = assertThrows(OMException.class,
+ () -> request.preExecute(ozoneManager));
+ assertEquals(OMException.ResultCodes.PERMISSION_DENIED, ex.getResult());
+ }
+
+ @Test
+ public void testPreExecuteNativeAuthorizerAllowsAdmin() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ IAccessAuthorizer authorizer = mock(IAccessAuthorizer.class);
+ when(authorizer.isNative()).thenReturn(true);
+ when(ozoneManager.getAccessAuthorizer()).thenReturn(authorizer);
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(true);
+
+ OMRequest omRequest =
+ setLifecycleConfigurationRequest(volumeName, bucketName, "ownerName");
+ OMLifecycleConfigurationSetRequest request =
+ new OMLifecycleConfigurationSetRequest(omRequest);
+ request.setUGI(UserGroupInformation.createRemoteUser("adminUser"));
+
+ assertNotNull(request.preExecute(ozoneManager));
+ }
+
+ @Test
+ public void testPreExecuteNativeAuthorizerAllowsOwner() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ IAccessAuthorizer authorizer = mock(IAccessAuthorizer.class);
+ when(authorizer.isNative()).thenReturn(true);
+ when(ozoneManager.getAccessAuthorizer()).thenReturn(authorizer);
+ when(ozoneManager.getBucketOwner(eq(volumeName), eq(bucketName),
+ any(IAccessAuthorizer.ACLType.class), any(OzoneObj.ResourceType.class)))
+ .thenReturn("bucketOwner");
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false);
+ when(ozoneManager.isOwner(any(UserGroupInformation.class), eq("bucketOwner")))
+ .thenReturn(true);
+
+ OMRequest omRequest =
+ setLifecycleConfigurationRequest(volumeName, bucketName, "ownerName");
+ OMLifecycleConfigurationSetRequest request =
+ new OMLifecycleConfigurationSetRequest(omRequest);
+ request.setUGI(UserGroupInformation.createRemoteUser("ownerUser"));
+
+ assertNotNull(request.preExecute(ozoneManager));
+ }
+
+ @Test
+ public void testDisallowSetLifecycleConfigurationBeforeFinalization() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
- LayoutVersionManager versionManager = mock(LayoutVersionManager.class);
- when(versionManager.isAllowed(OMLayoutFeature.S3_LIFECYCLE_SUPPORT)).thenReturn(false);
-
++ OMVersionManager versionManager = OMVersionManagerTestUtils.mockPreFinalizedOmVersionManager();
+ ValidationContext ctx = of(versionManager, omMetadataManager);
+ OMRequest request = setLifecycleConfigurationRequest(volumeName, bucketName, "ownerName");
+
+ OMException ex = assertThrows(OMException.class, () ->
+ OMLifecycleConfigurationSetRequest
+ .disallowSetLifecycleConfigurationBeforeFinalization(request, ctx));
+
+ assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION,
+ ex.getResult());
+ }
+
+ @Test
+ public void testAllowSetLifecycleConfigurationAfterFinalization() throws Exception {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
- LayoutVersionManager versionManager = mock(LayoutVersionManager.class);
- when(versionManager.isAllowed(OMLayoutFeature.S3_LIFECYCLE_SUPPORT)).thenReturn(true);
-
++ OMVersionManager versionManager = OMVersionManagerTestUtils.mockFinalizedOmVersionManager();
+ ValidationContext ctx = of(versionManager, omMetadataManager);
+ OMRequest request = setLifecycleConfigurationRequest(volumeName, bucketName, "ownerName");
+
+ OMRequest result = OMLifecycleConfigurationSetRequest
+ .disallowSetLifecycleConfigurationBeforeFinalization(request, ctx);
+
+ assertEquals(request, result);
+ }
+ }
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java
index 00000000000,8c5a806ac94..2bbd6f7ab01
mode 000000,100644..100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java
@@@ -1,0 -1,118 +1,117 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.ozone.om.request.lifecycle;
+
-import static org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.maxLayoutVersion;
+ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+ import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertNotNull;
+ import static org.junit.jupiter.api.Assertions.assertThrows;
+ import static org.junit.jupiter.api.Assertions.assertTrue;
+ import static org.mockito.ArgumentMatchers.any;
+ import static org.mockito.Mockito.mock;
+ import static org.mockito.Mockito.when;
+
+ import java.util.UUID;
+ import org.apache.hadoop.hdds.utils.db.Table;
+ import org.apache.hadoop.ozone.om.OMMetadataManager;
+ import org.apache.hadoop.ozone.om.OzoneManager;
+ import org.apache.hadoop.ozone.om.exceptions.OMException;
+ import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
+ import org.apache.hadoop.ozone.om.response.OMClientResponse;
-import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
++import org.apache.hadoop.ozone.om.upgrade.OMVersionManager;
++import org.apache.hadoop.ozone.om.upgrade.OMVersionManagerTestUtils;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleScanState;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest;
+ import org.apache.hadoop.security.UserGroupInformation;
+ import org.junit.jupiter.api.Test;
+
+ /**
+ * Tests OMLifecycleSaveScanStateRequest.
+ */
+ public class TestOMLifecycleSaveScanStateRequest {
+
+ @Test
+ public void testPreExecuteAdminCheck() throws Exception {
+ OzoneManager ozoneManager = mock(OzoneManager.class);
- OMLayoutVersionManager versionManager = mock(OMLayoutVersionManager.class);
- when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
++ OMVersionManager versionManager = OMVersionManagerTestUtils.mockPreFinalizedOmVersionManager();
+ when(ozoneManager.getVersionManager()).thenReturn(versionManager);
+
+ // Test when ACLs are enabled but user is not admin
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false);
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState)
+ .setClientId(UUID.randomUUID().toString())
+ .setSaveLifecycleScanStateRequest(SaveLifecycleScanStateRequest.newBuilder()
+ .setState(LifecycleScanState.newBuilder().setBucketKey("dummy").setScanStartTime(1L).build())
+ .build())
+ .build();
+
+ OMLifecycleSaveScanStateRequest request = new OMLifecycleSaveScanStateRequest(omRequest);
+ request.setUGI(UserGroupInformation.getCurrentUser());
+
+ OMException exception = assertThrows(OMException.class, () -> {
+ request.preExecute(ozoneManager);
+ });
+
+ assertEquals(OMException.ResultCodes.ACCESS_DENIED, exception.getResult());
+ assertTrue(exception.getMessage().contains("Superuser privilege is required"));
+
+ // Test when user is admin
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(true);
+ assertDoesNotThrow(() -> request.preExecute(ozoneManager));
+
+ // Test when ACLs are disabled
+ when(ozoneManager.getAclsEnabled()).thenReturn(false);
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false);
+ assertDoesNotThrow(() -> request.preExecute(ozoneManager));
+ }
+
+ @Test
+ public void testValidateAndUpdateCache() throws Exception {
+ OzoneManager ozoneManager = mock(OzoneManager.class);
+ OMMetadataManager omMetadataManager = mock(OMMetadataManager.class);
+ when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
+ Table<String, OmLifecycleScanState> table = mock(Table.class);
+ when(omMetadataManager.getLifecycleScanStateTable()).thenReturn(table);
+
+ LifecycleScanState stateProto = LifecycleScanState.newBuilder()
+ .setBucketKey("/vol1/bucket1")
+ .setScanStartTime(123456789L)
+ .setLastScannedKey("key1")
+ .build();
+
+ SaveLifecycleScanStateRequest saveRequest = SaveLifecycleScanStateRequest.newBuilder()
+ .setState(stateProto)
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState)
+ .setClientId(UUID.randomUUID().toString())
+ .setSaveLifecycleScanStateRequest(saveRequest)
+ .build();
+
+ OMLifecycleSaveScanStateRequest request = new OMLifecycleSaveScanStateRequest(omRequest);
+ OMClientResponse response = request.validateAndUpdateCache(ozoneManager, 100L);
+ assertNotNull(response);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK, response.getOMResponse().getStatus());
+ }
+ }
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java
index c8d8f51365e,08e8487ee08..f69afc7e9b0
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartRequestTests.java
@@@ -47,10 -49,12 +49,13 @@@ import org.apache.hadoop.ozone.om.Ozone
import org.apache.hadoop.ozone.om.ResolvedBucket;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.KeyValueUtil;
+ import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+ import org.apache.hadoop.ozone.om.response.OMClientResponse;
+ import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
-import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
+import org.apache.hadoop.ozone.om.upgrade.OMVersionManager;
+import org.apache.hadoop.ozone.om.upgrade.OMVersionManagerTestUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocation;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@@ -109,8 -113,14 +114,13 @@@ public class S3MultipartRequestTests
args.getVolumeName(), args.getBucketName(),
"owner", BucketLayout.DEFAULT);
});
- OMVersionManager versionManager = OMVersionManagerTestUtils.mockFinalizedOmVersionManager();
+ // MPU request tests default to a pre-finalized layout version, i.e. a
+ // cluster that has not finalized MPU_PARTS_TABLE_SPLIT. Newly initiated
+ // uploads therefore use the legacy (schema 0) inline parts layout. Tests
+ // that need the finalized behaviour (split parts table, schema 1) opt in
+ // by calling finalizeMpuPartsTableSplit().
- OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
- when(lvm.getMetadataLayoutVersion()).thenReturn(0);
- when(ozoneManager.getVersionManager()).thenReturn(lvm);
++ OMVersionManager versionManager = OMVersionManagerTestUtils.mockPreFinalizedOmVersionManager();
+ when(ozoneManager.getVersionManager()).thenReturn(versionManager);
when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
when(ozoneManager.getConfig()).thenReturn(ozoneConfiguration.getObject(OmConfig.class));
}
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManagerTestUtils.java
index d2549431274,00000000000..a03f8fd0eda
mode 100644,000000..100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManagerTestUtils.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManagerTestUtils.java
@@@ -1,48 -1,0 +1,63 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.upgrade;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.ozone.OzoneManagerVersion;
+
+/**
+ * Mockito helpers for {@link OMVersionManager} in unit tests.
+ */
+public final class OMVersionManagerTestUtils {
+
+ private OMVersionManagerTestUtils() {
+ }
+
+ /**
+ * Mock with apparent and software version both {@link OzoneManagerVersion#SOFTWARE_VERSION},
+ * {@link OMVersionManager#needsFinalization()} false, and {@link OMVersionManager#isAllowed(ComponentVersion)}
+ * true for any argument.
+ */
+ public static OMVersionManager mockFinalizedOmVersionManager() {
+ OMVersionManager ovm = mock(OMVersionManager.class);
+ when(ovm.getApparentVersion()).thenReturn(OzoneManagerVersion.SOFTWARE_VERSION);
+ when(ovm.getSoftwareVersion()).thenReturn(OzoneManagerVersion.SOFTWARE_VERSION);
+ when(ovm.needsFinalization()).thenReturn(false);
+ when(ovm.isAllowed(any(ComponentVersion.class))).thenReturn(true);
+ return ovm;
+ }
++
++ /**
++ * Mock with software version {@link OzoneManagerVersion#SOFTWARE_VERSION} and apparent version
++ * {@link OMLayoutFeature#INITIAL_VERSION},
++ * {@link OMVersionManager#needsFinalization()} true, and {@link OMVersionManager#isAllowed(ComponentVersion)}
++ * false for any argument.
++ */
++ public static OMVersionManager mockPreFinalizedOmVersionManager() {
++ OMVersionManager ovm = mock(OMVersionManager.class);
++ when(ovm.getApparentVersion()).thenReturn(OMLayoutFeature.INITIAL_VERSION);
++ when(ovm.getSoftwareVersion()).thenReturn(OzoneManagerVersion.SOFTWARE_VERSION);
++ when(ovm.needsFinalization()).thenReturn(true);
++ when(ovm.isAllowed(any(ComponentVersion.class))).thenReturn(false);
++ return ovm;
++ }
+}
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java
index e27bba5aadd,00000000000..1eba3fbc3c6
mode 100644,000000..100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java
@@@ -1,109 -1,0 +1,109 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.upgrade;
+
+import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy;
+import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests invariants for legacy OM layout feature versions.
+ */
+public class TestOMLayoutFeature {
+ @Test
+ public void testOMLayoutFeaturesHaveIncreasingLayoutVersion() {
+ OMLayoutFeature[] values = OMLayoutFeature.values();
+ int currVersion = -1;
+ for (OMLayoutFeature lf : values) {
+ // This will skip the jump from the last OMLayoutFeature to OzoneManagerVersion#ZDU,
+ // since that is expected to be a larger version increment.
+ assertEquals(currVersion + 1, lf.serialize(),
+ "Expected monotonically increasing layout version for " + lf);
+ currVersion = lf.serialize();
+ }
+ }
+
+ /**
+ * All incompatible changes to OM should now be added to {@link OzoneManagerVersion}.
+ */
+ @Test
+ public void testNoNewOMLayoutFeaturesAdded() {
+ int numOMLayoutFeatures = OMLayoutFeature.values().length;
+ OMLayoutFeature lastFeature = OMLayoutFeature.values()[numOMLayoutFeatures - 1];
- assertEquals(10, numOMLayoutFeatures);
- assertEquals(OMLayoutFeature.SNAPSHOT_DEFRAG, lastFeature);
- assertEquals(9, lastFeature.serialize());
++ assertEquals(12, numOMLayoutFeatures);
++ assertEquals(OMLayoutFeature.MPU_PARTS_TABLE_SPLIT, lastFeature);
++ assertEquals(11, lastFeature.serialize());
+ }
+
+ @Test
+ public void testNextVersion() {
+ OMLayoutFeature[] values = OMLayoutFeature.values();
+ for (int i = 1; i < values.length; i++) {
+ OMLayoutFeature previous = values[i - 1];
+ OMLayoutFeature current = values[i];
+ assertEquals(current, previous.nextVersion(),
+ "Expected " + previous + ".nextVersion() to be " + current);
+ }
+ // The last layout feature should point us to the ZDU version to switch to using OzoneManagerVersion.
+ assertEquals(OzoneManagerVersion.ZDU, values[values.length - 1].nextVersion());
+ }
+
+ @Test
+ public void testSerDes() {
+ for (OMLayoutFeature version : OMLayoutFeature.values()) {
+ assertEquals(version, OMLayoutFeature.deserialize(version.serialize()));
+ }
+ }
+
+ @Test
+ public void testDeserializeUnknownVersionReturnsNull() {
+ assertNull(OMLayoutFeature.deserialize(-1));
+ assertNull(OMLayoutFeature.deserialize(Integer.MAX_VALUE));
+ // OMLayoutFeature can only deserialize values from its own enum.
+ assertNull(OMLayoutFeature.deserialize(OzoneManagerVersion.ZDU.serialize()));
+ }
+
+ @Test
+ public void testIsSupportedByFeatureBoundary() {
+ for (OMLayoutFeature feature : OMLayoutFeature.values()) {
+ // A layout feature should support itself.
+ int layoutVersion = feature.serialize();
+ assertSupportedBy(feature, feature);
+ if (layoutVersion > 0) {
+ // A layout feature should not be supported by older features.
+ OMLayoutFeature previousFeature = OMLayoutFeature.values()[layoutVersion - 1];
+ assertNotSupportedBy(feature, previousFeature);
+ }
+ }
+ }
+
+ @Test
+ public void testAllLayoutFeaturesAreSupportedByUnknownFutureVersions() {
+ for (OMLayoutFeature feature : OMLayoutFeature.values()) {
+ assertSupportedBy(feature, OzoneManagerVersion.ZDU);
+ assertSupportedBy(feature, OzoneManagerVersion.UNKNOWN_VERSION);
+ // No ComponentVersion instance represents an arbitrary unknown future version.
+ assertTrue(feature.isSupportedBy(Integer.MAX_VALUE));
+ }
+ }
+}
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java
index 9531942e9c1,40e9e18e9bc..61a6542e02d
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java
@@@ -47,7 -43,7 +47,8 @@@ import org.apache.hadoop.ozone.om.helpe
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
-import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
+import org.apache.hadoop.ozone.om.upgrade.OMVersionManager;
++import org.apache.hadoop.ozone.om.upgrade.OMVersionManagerTestUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.util.Time;
@@@ -587,4 -481,69 +588,67 @@@ public class TestOzoneManagerRequestHan
reportOnlyResponse.getSnapshotDiffResponse().getJobStatus());
Assertions.assertEquals(0L, reportOnlyResponse.getSnapshotDiffResponse().getWaitTimeInMs());
}
+
+ @Test
+ public void testSnapshotDiffHandlerSerializesSubStatusAndProgressIntoProto() throws IOException {
+ OzoneManagerRequestHandler handler = getRequestHandler(10);
+ OzoneManager ozoneManager = handler.getOzoneManager();
+
- OMLayoutVersionManager lvm = Mockito.mock(OMLayoutVersionManager.class);
- Mockito.when(lvm.isAllowed(Mockito.anyString())).thenReturn(true);
- Mockito.when(ozoneManager.getVersionManager()).thenReturn(lvm);
++ OMVersionManager versionManager = OMVersionManagerTestUtils.mockFinalizedOmVersionManager();
++ Mockito.when(ozoneManager.getVersionManager()).thenReturn(versionManager);
+
+ SnapshotDiffResponse diffResponse =
+ new SnapshotDiffResponse(null, SnapshotDiffResponse.JobStatus.IN_PROGRESS, 60000L);
+ diffResponse.setSubStatus(SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_FSO_DIR);
+ diffResponse.setProgressPercent(50.0);
+ Mockito.when(ozoneManager.snapshotDiff(Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt()))
+ .thenReturn(diffResponse);
+
+ OzoneManagerProtocolProtos.OMRequest request =
+ OzoneManagerProtocolProtos.OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SnapshotDiff)
+ .setClientId("client")
+ .setSnapshotDiffRequest(OzoneManagerProtocolProtos.SnapshotDiffRequest.newBuilder()
+ .setVolumeName("vol").setBucketName("buck")
+ .setFromSnapshot("s1").setToSnapshot("s2")
+ .setToken("t").setPageSize(10))
+ .build();
+
+ OzoneManagerProtocolProtos.SnapshotDiffResponse proto =
+ handler.handleReadRequest(request).getSnapshotDiffResponse();
+ Assertions.assertTrue(proto.hasSubStatus());
+ Assertions.assertEquals(
+ OzoneManagerProtocolProtos.SnapshotDiffResponse.SubStatus.OBJECT_ID_MAP_GEN_FSO_DIR,
+ proto.getSubStatus());
+ Assertions.assertEquals(50.0, proto.getProgressPercent(), 1e-9);
+ }
+
+ @Test
+ public void testSnapshotDiffHandlerOmitsSubStatusFromProtoWhenNull() throws IOException {
+ OzoneManagerRequestHandler handler = getRequestHandler(10);
+ OzoneManager ozoneManager = handler.getOzoneManager();
+
- OMLayoutVersionManager lvm = Mockito.mock(OMLayoutVersionManager.class);
- Mockito.when(lvm.isAllowed(Mockito.anyString())).thenReturn(true);
- Mockito.when(ozoneManager.getVersionManager()).thenReturn(lvm);
++ OMVersionManager versionManager = OMVersionManagerTestUtils.mockFinalizedOmVersionManager();
++ Mockito.when(ozoneManager.getVersionManager()).thenReturn(versionManager);
+
+ Mockito.when(ozoneManager.snapshotDiff(Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt()))
+ .thenReturn(new SnapshotDiffResponse(null, SnapshotDiffResponse.JobStatus.IN_PROGRESS, 60000L));
+
+ OzoneManagerProtocolProtos.OMRequest request =
+ OzoneManagerProtocolProtos.OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SnapshotDiff)
+ .setClientId("client")
+ .setSnapshotDiffRequest(OzoneManagerProtocolProtos.SnapshotDiffRequest.newBuilder()
+ .setVolumeName("vol").setBucketName("buck")
+ .setFromSnapshot("s1").setToSnapshot("s2")
+ .setToken("t").setPageSize(10))
+ .build();
+
+ OzoneManagerProtocolProtos.SnapshotDiffResponse proto =
+ handler.handleReadRequest(request).getSnapshotDiffResponse();
+ Assertions.assertFalse(proto.hasSubStatus());
+ Assertions.assertFalse(proto.hasProgressPercent());
+ }
}
diff --cc hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java
index e0336664b52,67aa0e9d569..9d18daa0fcf
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconUpgradeAction.java
@@@ -17,97 -17,15 +17,91 @@@
package org.apache.hadoop.ozone.recon.upgrade;
+import static org.apache.ozone.recon.schema.ContainerSchemaDefinition.UNHEALTHY_CONTAINERS_TABLE_NAME;
+import static org.apache.ozone.recon.schema.SqlDbUtils.TABLE_EXISTS_CHECK;
+import static org.jooq.impl.DSL.field;
+
+import com.google.inject.Injector;
+import java.sql.Connection;
+import java.sql.SQLException;
- import java.util.Arrays;
import javax.sql.DataSource;
+import org.apache.hadoop.ozone.recon.ReconGuiceServletContextListener;
+import org.apache.hadoop.ozone.recon.tasks.NSSummaryTask;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskController;
+import org.apache.hadoop.ozone.recon.tasks.ReconTaskReInitializationEvent;
+import org.apache.hadoop.ozone.upgrade.UpgradeAction;
+import org.apache.ozone.recon.schema.ContainerSchemaDefinition;
+import org.jooq.DSLContext;
+import org.jooq.exception.DataAccessException;
+import org.jooq.impl.DSL;
+import org.slf4j.Logger;
/**
- * ReconUpgradeAction is an interface for executing upgrade actions in Recon.
+ * Recon upgrade action executed during finalization of a {@link ReconVersion}.
*/
-public interface ReconUpgradeAction {
-
+public interface ReconUpgradeAction extends UpgradeAction<DataSource> {
/**
* Execute the upgrade action during finalization.
*/
+ @Override
void execute(DataSource source) throws Exception;
+
+ /**
+ * Helper method used by upgrade actions that need to add new unhealthy container states to the unhealthy containers
+ * table.
+ */
+ static void updateUnhealthyContainerStatesConstraint(DataSource dataSource, Logger log) throws SQLException {
+ try (Connection conn = dataSource.getConnection()) {
+ if (!TABLE_EXISTS_CHECK.test(conn, UNHEALTHY_CONTAINERS_TABLE_NAME)) {
+ return;
+ }
+ DSLContext dslContext = DSL.using(conn);
+ final String constraintName = UNHEALTHY_CONTAINERS_TABLE_NAME + "ck1";
+
+ try {
+ dslContext.alterTable(UNHEALTHY_CONTAINERS_TABLE_NAME)
+ .dropConstraint(constraintName)
+ .execute();
+ log.debug("Dropped the existing constraint: {}", constraintName);
+ } catch (DataAccessException e) {
+ log.debug("Constraint {} was not present: {}", constraintName, e.getMessage());
+ }
-
- String[] enumStates = Arrays
- .stream(ContainerSchemaDefinition.UnHealthyContainerStates.values())
- .map(Enum::name)
- .toArray(String[]::new);
-
+ dslContext.alterTable(UNHEALTHY_CONTAINERS_TABLE_NAME)
+ .add(DSL.constraint(constraintName)
- .check(field(DSL.name("container_state")).in(enumStates)))
++ .check(field(DSL.name("container_state"))
++ .in(ContainerSchemaDefinition.UnHealthyContainerStates.NAMES)))
+ .execute();
+
+ log.info("Added the updated constraint to the UNHEALTHY_CONTAINERS table for enum state values: {}",
- Arrays.toString(enumStates));
++ ContainerSchemaDefinition.UnHealthyContainerStates.NAMES);
+ }
+ }
+
+ /**
+ * Idempotently queues an NSSummary rebuild, skipping if a rebuild is already in progress.
+ * Logs and returns without throwing if the queue attempt does not succeed.
+ */
+ static void queueNSSummaryRebuildIfNeeded(Logger log) {
+ Injector injector = ReconGuiceServletContextListener.getGlobalInjector();
+ if (injector == null) {
+ throw new IllegalStateException(
+ "Guice injector not initialized. NSSummary rebuild cannot proceed during upgrade.");
+ }
+
+ ReconTaskController reconTaskController = injector.getInstance(ReconTaskController.class);
+
+
+ log.info("Triggering asynchronous NSSummary tree rebuild.");
+ if (NSSummaryTask.getRebuildState() == NSSummaryTask.RebuildState.RUNNING) {
+ log.info("NSSummary rebuild already in progress, skipping duplicate upgrade queue.");
+ return;
+ }
+ ReconTaskController.ReInitializationResult result = reconTaskController.queueReInitializationEvent(
+ ReconTaskReInitializationEvent.ReInitializationReason.MANUAL_TRIGGER);
+ if (result != ReconTaskController.ReInitializationResult.SUCCESS) {
+ log.error(
+ "Failed to queue reinitialization event for NSSummary rebuild (result: {}). "
+ + "Will be retried as part of syncDataFromOM scheduler task.", result);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org