You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2021/04/14 11:59:10 UTC
[ozone] branch HDDS-4944-S3-Multitenant-Users updated (45eee3f ->
54ea4c7)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a change to branch HDDS-4944-S3-Multitenant-Users
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 45eee3f HDDS-4919. Fix spelling (#2004)
add 95739ea HDDS-3185 Construct a standalone ratis server for SCM. (#720)
add 31c7386 HDDS-3187 Construct SCM StateMachine. (#819)
add 8f2107a Resolve conflicts with merge from master.
add aa2884c HDDS-3556 Refactor conf in SCMRatisServer to Java-based conf. (#907)
add 1f3ef36 HDDS-3186. Introduce generic SCMRatisRequest and SCMRatisResponse. (#959)
add 30e1751 HDDS-3192. Handle AllocateContainer operation for HA. (#975)
add c836720 HDDS-3196 New PipelineManager interface to persist to RatisServer. (#980)
add 988b23a HDDS-3693 Switch to new StateManager interface. (#1007)
add 5355939 HDDS-3711. Handle inner classes in SCMRatisRequest and SCMRatisResponse. (#1016)
add 8e86480 HDDS-3679 Add tests for PipelineManager V2. (#1019)
add 8d74c0c HDDS-3652 Add test for SCMRatisResponse. (#1113)
add 3e7c427 Merge branch 'master' into HDDS-2823
add 82c30a4 Merge branch 'master' into HDDS-2823
add 7287e1d HDDS-3651 Add tests for SCMRatisRequest. (#1112)
add 144f9a8 HDDS-3911. Compile error in acceptance test on HDDS-2823 (#1157)
add 565dabc HDDS-3662 Decouple finalizeAndDestroyPipeline. (#1049)
add 8a8c9eb HDDS-3191: switch from SCMPipelineManager to PipelineManagerV2Impl (#1151)
add 40127b3 Merge branch 'master' into HDDS-2823
add 6de98c6 Merge branch 'master' into HDDS-2823
add 58394eb HDDS-3837. Add isLeader check in SCMHAManager.
add 3ed29d8 HDDS-4059. SCMStateMachine::applyTransaction() should not invoke TransactionContext.getClientRequest().
add d482abf HDDS-4125. Pipeline is not removed when a datanode goes stale.
add a70964e HDDS-4130. remove the 1st edition of RatisServer of SCM HA which is copied from OM HA.
add 9e0dd84 HDDS-3895. Implement container related operations in ContainerManagerImpl.
add 5f3981c HDDS-4115. CLI command to show current SCM leader and follower status.
add 9f7ab46 HDDS-3188. Add failover proxy for SCM block location.
add 5111126 HDDS-4192. enable SCM Raft Group based on config ozone.scm.names.
add 43b87fe HDDS-4365. SCMBlockLocationFailoverProxyProvider should use ScmBlockLocationProtocolPB.class in RPC.setProtocolEngine.
add 782057a Resolving master merge conflict.
add 44a6503 HDDS-4393. Addressing test failures after master merge. (#1587)
add f30bc4e Merge branch 'master' into HDDS-2823
add 285d793 HDDS-4191 Add failover proxy for SCM container location. (#1514)
add 48b9809 HDDS-4538: Workaround on HDDS-2823, hard code scmUuid and clusterID. (#1649)
add 0801203 HDDS-4542. Need throw exception to trigger FailoverProxyProvider of SCM client to work (#1652)
add 0aa9ba3 HDDS-3988: DN can distinguish SCMCommand from stale leader SCM (#1314)
add 34c393c HDDS-4551: Remove checkLeader in PipelineManager. (#1658)
add 91eb342 HDDS-4560. Add ReadWriteLock into PipelineStateManagerV2Impl to protect contentions between RaftServer and PipelineManager. (#1676)
add 8b16a35 HDDS-4133. Use new ContainerManager in SCM. (#1378)
add 2c3aed3 Merge branch 'master' into HDDS-2823
add 7f5e522 HDDS-4575: Refactor SCMHAManager and SCMRatisServer with RaftServer.Division (#1683)
add 9e9defd HDDS-4589: Handle potential data loss during ReplicationManager.handleOverReplicatedContainer(). (#1700)
add adbc89b HDDS-4628: min/max election timeout of SCMRatisServer is not set properly. (#1742)
add eb159d3 HDDS-4632: Add term into SetNodeOperationalStateCommand. (#1745)
add f64a1ce HDDS-4624. Fix set configs in SCMHAConfigration (#1739)
add 13f2ece HDDS-3684. Add tests for replication annotation (#1650)
add b32c2bb HDDS-4643. Ratis Snapshot should be loaded from the config (#1756)
add b0d79e7 Merge branch 'master' into HDDS-2823
add 1a04bbb HDDS-4130: remove the left part of the 1st edition of RatisServer of SCM HA which is copied from OM HA (#1760)
add a52e8ef HDDS-4630: Solve deadlock triggered by PipelineActionHandler. (#1743)
add a5906e6 HDDS-4622: Use singe server raft cluster in MiniOzoneCluster. (#1744)
add 572a7de HDDS-4533.Avoid rewriting pipeline information during PipelineStateManagerV2Impl initialization (#1796)
add 47aa782 HDDS-3212. Allow enabling purge SCM Ratis log (#1802)
add c957e2b HDDS-4532. Update pipeline db when pipeline state is changed. (#1785)
add bb9c68f HDDS-4568. Add SCMContext to SCM HA (#1737)
add cb79d30 HDDS-4651. Implement a sequence ID generator (#1810)
add 3ff677d HDDS-3208. Implement Ratis snapshot on SCM (#1725)
add 8ce33f2 HDDS-4695. Support encode and decode ArrayList and Long. (#1831)
add 3b6918e HDDS-4295. Add SCMServiceManager to SCM HA. (#1784)
add b328608 HDDS-3205. Handle BlockDeletingService in SCM HA (#1780)
add 66ba5bc Merge branch 'master' into HDDS-2823
add 5554023 Merge branch 'master' into HDDS-2823
add 2cb0e1e HDDS-4800. Fix TestContainerEndpoint after merging master to HDDS-2823. (#1903)
add b52ed77 HDDS-4806. Fix misc acceptance test: List pipelines on unknown host (#1907)
add 56bf321 HDDS-4804. Fix TestReconContainerManager after merge master to HDDS-2823 (#1905)
add 069a9b4 HDDS-4797: Fix findbugs issues after HDDS-2195 (#1902)
add d402f21 HDDS-4600. SCM HA Config Phase 1: Use SCMNodeDetails as the entry to initialize SCM except for Ratis servers (#1722)
add 4771661 HDDS-4782.Merge SCMRatisSnapshotInfo and OMRatisSnapshotInfo into a single class (#1880)
add 3a233ad HDDS-4778. Add transactionId into deletingTxIDs when remove it from DB (#1873)
add 97da84d HDDS-4823. Make SCM Generic config support HA Style. (#1924)
add 2cfd618 HDDS-4786. Disable Ratis new features to be consistent with master branch (#1899)
add 4c36f14 HDDS-4756. Add lock for activate/deactivate in PipelineManagerV2 (#1862)
add e6239c8 HDDS-4812. Move Ratis group creation to scm --init phase (#1916)
add b60a230 HDDS-4829. Rename MiniOzoneHACluster to MiniOzoneOMHACluster. (#1925)
add e5b281a HDDS-4660. Update OMTransactionInfo to TransactionInfo with functions added. (#1932)
add 1cbd6e3 HDDS-4773. Add functionality to transfer Rocks db checkpoint from leader to follower (#1870)
add 8d49b81 HDDS-4821. Use SCM service ID in SCMBlockClient and SCM Client. (#1940)
add 3a473e6 HDDS-4822. Implement scm --bootstrap command. (#1921)
add d17ff92 HDDS-4107. replace scmID with clusterID for container and volume at Datanode side. (#1947)
add 1334876 HDDS-4837. Use SCM service ID in finding SCM Datanode address. (#1934)
add 222be1c HDDS-4841. Make changes required for SCM admin commands to work with … (#1949)
add 8e56906 HDDS-4861. [SCM HA Security] Implement generate SCM certificate. (#1958)
add 920381c HDDS-4718. Bootstrap new SCM node (#1953)
add 590a9c5 HDDS-4740: Admin command should take effect on all SCM instance. (#1893)
add 762fb00 HDDS-4761. Implement increment count optimization in DeletedBlockLog V2 (#1914)
add 7257243 HDDS-4880. Fix removing local SCM when submitting request to other SCM. (#1971)
add 8f6941a Merge remote-tracking branch 'github/master' into HDDS-2823-2021-03-01 (#1972)
add d682164 HDDS-4884. Fix and enable TestReconTasks. (#1974)
add db25b54 HDDS-4874. [SCM HA Security] Implement listCertificates based on role (#1969)
add 1cf9f0d HDDS-4886. Fix and enable TestEndpoints.java. (#1975)
add 6c8fea8 HDDS-4866. Datanode with scmID format should work with clusterID dire… (#1959)
add 4a312ba HDDS-4807. Add install checkpoint in SCMStateMachine. (#1936)
add 00c52fb HDDS-4894. Use PipelineManagerV2Impl in Recon and enable ignored Recon test cases. (#1987)
add 538cb2d HDDS-4651: Distributed Sequence ID Gen. (#1980)
add faeb3d5 HDDS-4890. SCM Ratis enable/disable switch (#1981)
add 98ea082 HDDS-4863. Enable SCM HA in ozone-ha test (#1967)
add 14a4e87 HDDS-4912. Support inline upgrade from containerId, delTxnId, localId to SequenceIdGenerator. (#1996)
add 20b8bca HDDS-4876. [SCM HA Security] Add failover proxy to SCM Security Server Protocol (#1978)
add 40c738f HDDS-4896. Need a tool to upgrade current non-HA SCM node to single node HA cluster (#1999)
add 33d4351 HDDS-4922. refactor code in SCMStateMachine. (#2007)
add afeab2c HDDS-4951. Return with exit code 0 in case of optional scm bootstrap/init (#2022)
add ca9e76e HDDS-4957. Fix flaky test TestSCMInstallSnapshotWithHA#testInstallCorruptedCheckpointFailure (#2025)
add dd7750e HDDS-4877. Make SCM ratis server spin up time during initialization configurable (#2028)
add 2cb80b1 Revert " Merge remote-tracking branch 'github/master' into HDDS-2823-2021-03-01 (#1972)"
add b95eeba HDDS-4881. Pre-install awscli in ozone-runner (#2002)
add ec1dcd2 HDDS-4920. Rename Apache Hadoop Ozone to Apache Ozone in pom and markdown files (#2005)
add 1493541 HDDS-4891. Avoid latest/master in Github Actions (#1983)
add aae9ebd HDDS-4921. Replace the usage of deprecated Junit#timeout() in Ozone. (#2006)
add 2fc1022 HDDS-4905. Add prefix CLI. (#1995)
add 58aa3fc Merge branch 'master' into HDDS-2823-Merge-Master-0312-Fix
add 6105322 HDDS-4953. [SCM HA Security] Make CertStore DB updates for StoreValidateCertificate go via Ratis (#2034)
add 890b814 HDDS-4879. Support reserved space for single dir (#1970)
add 9670e35 HDDS-4911. List container by container state. (#2001)
add 2ce0594 HDDS-4947. Reuse compiled binaries in combined coverage calculation (#2020)
add fb7f18c HDDS-4972. Fix typo in SCMConnectionManager.java (#2033)
add b0fed25 HDDS-2981 Add unit tests for Proto [de]serialization (#1425)
add 456d5f9 Merge remote-tracking branch 'origin/master' into HDDS-4968
add be28d3f HDDS-4968. Back-port HDDS-4911 (List container by container state) to ContainerManagerV2
add dd7c1d6 HDDS-4985. [SCM HA Security] When Ratis enable, SCM secure cluster is not working. (#2052)
add acc1d21 HDDS-4978. [SCM HA Security] Ozone services should be disabled in SCM HA enabled and security enabled cluster (#2043)
add 2afba1f HDDS-4952. Implement listCAs and getRootCA API. (#2042)
add 0795be5 HDDS-4950. Provide example k8s files to run full HA Ozone (#2023)
add 5865873 HDDS-4899. Add SCM HA to Chaos tests. (#1989)
add 9e4bcb2 HDDS-4948. SCM-HA documentation (#2050)
add e2997ef HDDS-4998. [SCM HA Security] Make storeValidCertificate method idempotent (#2063)
add cdc170d HDDS-4897. [SCM HA Security] Create SCM Cert Client and change DefaultCA to allow selfsigned and intermediary (#2041)
add 5f20cd4 HDDS-5015. localId is not consistent across SCMs when setup a multi node SCM HA cluster. (#2072)
add d6e5b71 HDDS-4981. Remove SequenceIdGenerator#StateManagerImpl (#2079)
add 22c2d41 HDDS-4982.Solve intellj warnings on DBTransactionBuffer (#2082)
add 65e0609 HDDS-4971. Define Ozone specific trash-interval and checkpoint interval (#2031)
add 42a3719 HDDS-4975. Refine missing delete block failure log (#2039)
add 9fca2a7 HDDS-4933. Make CI checks fail faster (#2037)
add eab55e4 HDDS-4916. No ACL check when uploading object through multi-upload. (#2009)
add db3c50d HDDS-4395. Ozone Data Generator for Fast Scale Test (#1826)
add 79877fa HDDS-4984. Support async profiler 2.0 (#2048)
add 4136d47 HDDS-4552. Read data from chunk into ByteBuffer[] instead of single ByteBuffer. (#1685)
add cc3227c HDDS-4792. Remove redundant state check for handleDeleteChunk (#1894)
add 8d97993 HDDS-4802. Fix an internal variable always not null in PipelinePlacementPolicy (#1904)
add ae8d2e8 HDDS-4824. Remove repeated value set of reports in StateContext.addReport (#1920)
add a130578 HDDS-4814. Remove CMS OPTS for Java 15 (#2049)
add 85af50b HDDS-4996. Upgrade kotlin-stdlib (#2061)
add a41ffc4 HDDS-5006. Use Log4j to print error messages for Datanode startup (#2066)
add 226a41a HDDS-4856. Ruby S3 SDK never get authenticated by Ozone (#2013)
add 7b12f00 HDDS-4872. Upgrade UsageInfoSubcommand with options to show most and least used datanodes. (#1982)
add 3eefd47 HDDS-5020. Bump ratis version to 2.0.0 (#2076)
add 520ba00 HDDS-5010. Fixing Prometheus duplicated TYPE definition (#2074)
add 96467c3 HDDS-5024. Remove empty share/hadoop directory in dist package. (#2086)
add 378ea41 HDDS-4904. Remove mention of CSI support (#2036)
add ddf180f HDDS-4989. Decommission CLI should return details of nodes which fail (#2057)
add fda8656 Merge remote-tracking branch 'apache/master' into HDDS-2823
add 8d57331 Modify the version of ozone-runner (#2078)
add 685ff3f HDDS-4995. Skip CI for draft pull requests (#2060)
add a034028 HDDS-5007. Intellij run configuration for ozonefs shell. (#2081)
add 4b4b1df HDDS-5039. Support command changing the logging level of a server without restarting the server (#2091)
add d5bf345 HDDS-5019. Inconsistent tmp bucket name on different nodes when Kerberos is enabled (#2075)
add b75b8e3 HDDS-5049. Add timeout support for ratis requests in SCM HA. (#2099)
add 5bb8dda HDDS-4915. [SCM HA Security] Integrate CertClient. (#2000)
add 67c4c13 HDDS-4987. Import container should not delete container contents if container already exists (#2077)
add fcd0160 HDDS-4901. Remove OmOzoneAclMap from OmVolumeArgs to avoid OzoneAcl conversions (#1992)
add 3caca5b HDDS-3752. Fix o3fs list bucket contents issue when without tailing "/" (#2088)
add 1d4351e HDDS-5033. SCM may not be able to know full port list of Datanode after Datanode is started. (#2090)
add 722ae37 HDDS-5022. SCM get roles command should provide Ratis Leader/Follower… (#2098)
add b57041e HDDS-4553. ChunkInputStream should release buffer as soon as last byte in the buffer is read (#2062)
add 8ec5f43 HDDS-4506. Support query parameter based v4 auth in S3g (#1628)
add d100652 HDDS-5058. Make getScmInfo retry for a duration.
add 8d331a3 HDDS-4504. Datanode deletion config should be based on number of blocks (#1885)
add c0b40f7 HDDS-5032. DN stopped to load containers on volume after a container load exception. (#2109)
add 843d39e HDDS-5035. Use default config values to solve generated config file conflict (#2087)
add d1d4303 HDDS-5066. Use fixed vesion from pnpm to build recon (#2115)
add e5babd2 fix project name in NOTICE.txt (#2112)
add 68265b8 HDDS-4925. Introduce ContainerBalancer in SCM with start/stop capabilities. (#2097)
add ee4f759 HDDS-5011. Introduce Java based ReplicationConfig implementation (#2089)
add 33ddcb3 HDDS-5062. Add a config to bypass clusterId validation for bootstrapping SCM. (#2114)
add 6822b10 HDDS-5032. Fix findbugs (#2120)
add e40c638 HDDS-5027. [SCM HA Security] Handle leader changes during bootstrap. (#2113)
add 534eefa HDDS-5056. Avoid false positiver error messages during pipeline creations (#2105)
add abf38f4 HDDS-4988. Cancel duplicate PR workflows (#2053)
add a170a28 HDDS-5071. Fix duration of sub-ca certs. (#2127)
add 6b57765 HDDS-5052. [SCM HA Security] Handle leader changes between SCMInfo and getSCMSigned Cert in OM/SCM. (#2100)
add 4f76d03 HDDS-4969. Extract check dependency installation from Github Actions workflow (#2029)
add 9c92855 HDDS-4840. Make datanode db profile configurable with existing hdds.d… (#1955)
add 9de5d0f HDDS-5057. Remove backward direction dependency between HDDS->Ozone (#2106)
add 1a7745e HDDS-5004. Upgrade Jersey2 dependency (#2065)
add b83c7b6 HDDS-5038. Create and update ZH translation of SCM-HA.md and OM-HA.md in doc (#2092)
add ccd046e HDDS-5077. Include dev-support in ozone source distribution (#2134)
add 8ea5be9 HDDS-5050. Add retry policy for ratis requests in SCM HA. (#2116)
add c8594d1 HDDS-5053. RocksDB block cache capacity is wrongly configured (#2137)
add fd308b7 HDDS-4980. Bump jaeger-client to 1.6.0 (#2045)
add aa44905 HDDS-5074. Bump Guava version (#2131)
add f1be299 HDDS-5098. Disable intermittent TestSCMNodeManager#testScmDetectStaleAndDeadNode
add 5bebe62 HDDS-4842. Add timestamp to Revoked Certs table in SCM DB (#2145)
add 89a2b52 HDDS-5075. NPE during secure SCM initialization with HA code updated to an already existing cluster. (#2146)
add 3a09f6b HDDS-4642. SCM security protocol support for query CRLs and latest CRL id for OM and Datanode. (#2030)
add 54ea4c7 HDDS-5080. enable s3 test suite for secure-ha (#2147)
No new revisions were added by this update.
Summary of changes:
.../{close-pending.yaml => cancel-ci.yaml} | 28 +-
.github/workflows/close-pending.yaml | 4 +-
.github/workflows/comments.yaml | 8 +-
.github/workflows/post-commit.yml | 50 +-
.gitignore | 2 +
CONTRIBUTING.md | 14 +-
HISTORY.md | 2 +-
NOTICE.txt | 2 +-
hadoop-hdds/client/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 12 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 21 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 4 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 5 +
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 286 ++++++--
.../hdds/scm/storage/DummyChunkInputStream.java | 32 +-
.../hdds/scm/storage/TestChunkInputStream.java | 50 +-
hadoop-hdds/common/pom.xml | 12 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 80 ++-
.../hadoop/hdds/client/RatisReplicationConfig.java | 82 +++
.../hadoop/hdds/client/ReplicationConfig.java | 98 +++
.../hdds/client/StandaloneReplicationConfig.java | 72 ++
.../hadoop/hdds/protocol/DatanodeDetails.java | 8 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 4 +-
.../hdds/scm/{ScmInfo.java => AddSCMRequest.java} | 66 +-
.../scm/DatanodeAdminError.java} | 48 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 162 ++++-
.../java/org/apache/hadoop/hdds/scm/ScmInfo.java | 32 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 54 +-
.../hadoop/hdds/scm/container/ContainerID.java | 30 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 28 +-
.../scm/container/common/helpers/ExcludeList.java | 2 +-
.../hadoop/hdds/scm/exceptions/SCMException.java | 3 +-
.../scm/ha/RetriableWithNoFailoverException.java | 12 +-
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 320 +++++++++
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 242 +++++++
.../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 249 +++++++
.../apache/hadoop/hdds/scm/ha}/package-info.java | 9 +-
.../protocol/StorageContainerLocationProtocol.java | 69 +-
.../ContainerCommandResponseBuilders.java | 82 ++-
.../hdds/scm/storage/ContainerProtocolCalls.java | 4 +-
.../hadoop/hdds/scm/utils/ClientCommandsUtils.java | 47 ++
.../hadoop/hdds/scm/utils}/package-info.java | 7 +-
.../security/exception/SCMSecurityException.java | 43 +-
.../hadoop/hdds/security/x509/SecurityConfig.java | 13 +-
.../x509/certificate/utils/CertificateCodec.java | 6 +-
.../org/apache/hadoop/hdds/server/ServerUtils.java | 0
.../apache/hadoop/hdds/server}/package-info.java | 9 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 33 +-
.../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 25 +
.../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 48 ++
.../apache/hadoop/ozone/common/ChunkBuffer.java | 14 +
.../common/ChunkBufferImplWithByteBuffer.java | 7 +
.../common/ChunkBufferImplWithByteBufferList.java | 47 +-
.../ozone/common/IncrementalChunkBuffer.java | 16 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 3 +-
.../ozone/common/ha/ratis/RatisSnapshotInfo.java | 17 +-
.../ha/ratis/package-info.java} | 38 +-
.../hadoop/ozone/common/utils/BufferUtils.java | 133 ++++
.../hadoop/ozone/common/utils}/package-info.java | 7 +-
.../ozone/container/common/helpers/ChunkInfo.java | 12 +
.../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 112 +++
.../org/apache/hadoop/ozone/ha/package-info.java | 21 +
.../common/src/main/resources/ozone-default.xml | 275 +++++++-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 46 +-
.../conf/TestGeneratedConfigurationOverwrite.java | 66 ++
.../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 163 +++++
.../apache/hadoop/hdds/scm/ha}/package-info.java | 5 +-
.../hdds/scm/net/TestNetworkTopologyImpl.java | 2 +-
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 4 +-
.../hadoop/hdds/scm/net/TestNodeSchemaManager.java | 2 +-
.../hadoop/hdds/scm/net/TestYamlSchemaLoader.java | 2 +-
hadoop-hdds/config/pom.xml | 4 +-
.../java/org/apache/hadoop/hdds/conf/Config.java | 5 +
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 4 +-
.../hdds/conf/ConfigurationReflectionUtil.java | 25 +-
hadoop-hdds/container-service/pom.xml | 4 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 15 +-
.../container/common/helpers/ContainerUtils.java | 16 +-
.../container/common/impl/HddsDispatcher.java | 12 +-
.../RandomContainerDeletionChoosingPolicy.java | 35 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 45 +-
.../ContainerDeletionChoosingPolicy.java | 5 +-
.../common/interfaces/ContainerDispatcher.java | 2 +-
.../ozone/container/common/interfaces/Handler.java | 6 +-
.../common/statemachine/DatanodeConfiguration.java | 17 +
.../common/statemachine/SCMConnectionManager.java | 4 +-
.../common/statemachine/StateContext.java | 118 +++-
.../states/endpoint/HeartbeatEndpointTask.java | 22 +
.../states/endpoint/VersionEndpointTask.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 16 +-
.../transport/server/ratis/XceiverServerRatis.java | 13 +-
.../container/common/utils/HddsVolumeUtil.java | 23 +-
.../ozone/container/common/volume/HddsVolume.java | 4 +-
.../container/common/volume/MutableVolumeSet.java | 25 +-
.../ozone/container/common/volume/VolumeInfo.java | 36 +-
.../container/keyvalue/KeyValueContainer.java | 27 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 64 +-
.../container/keyvalue/helpers/ChunkUtils.java | 11 +-
.../helpers/KeyValueContainerLocationUtil.java | 23 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 75 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 2 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 20 +-
.../keyvalue/impl/FilePerChunkStrategy.java | 17 +-
.../keyvalue/interfaces/BlockManager.java | 2 +
.../keyvalue/interfaces/ChunkManager.java | 31 +
.../background/BlockDeletingService.java | 111 +--
.../container/metadata/AbstractDatanodeStore.java | 26 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 165 ++++-
.../ozone/container/ozoneimpl/OzoneContainer.java | 52 +-
.../hadoop/ozone/protocol/commands/SCMCommand.java | 20 +-
.../ozone/container/ContainerTestHelper.java | 1 +
.../hadoop/ozone/container/common/ScmTestMock.java | 16 +-
.../container/common/TestBlockDeletingService.java | 86 ++-
.../container/common/helpers/TestBlockData.java | 2 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 66 +-
.../common/impl/TestContainerPersistence.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 14 +-
.../container/common/interfaces/TestHandler.java | 2 +-
.../container/common/volume/TestHddsVolume.java | 13 +-
.../common/volume/TestHddsVolumeChecker.java | 2 +-
.../container/common/volume/TestVolumeSet.java | 2 +-
.../common/volume/TestVolumeSetDiskChecks.java | 2 +-
.../container/keyvalue/ChunkLayoutTestInfo.java | 2 +-
.../container/keyvalue/TestKeyValueContainer.java | 68 +-
.../TestKeyValueContainerMarkUnhealthy.java | 2 +-
.../container/keyvalue/TestKeyValueHandler.java | 4 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 3 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 24 +-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 9 +-
.../container/ozoneimpl/TestContainerReader.java | 53 +-
.../container/ozoneimpl/TestOzoneContainer.java | 10 +-
hadoop-hdds/docs/content/design/scmha.md | 15 +-
.../docs/content/feature/{HA.md => OM-HA.md} | 18 +-
.../docs/content/feature/{HA.zh.md => OM-HA.zh.md} | 13 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 162 +++++
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 162 +++++
hadoop-hdds/docs/content/interface/CSI.md | 5 +
hadoop-hdds/docs/content/interface/S3.md | 3 +-
hadoop-hdds/docs/static/ozone-usage.png | Bin 104961 -> 120495 bytes
hadoop-hdds/framework/pom.xml | 4 +-
.../java/org/apache/hadoop/hdds}/ExitManager.java | 2 +-
.../java/org/apache/hadoop/hdds/NodeDetails.java | 103 +++
.../java/org/apache/hadoop/hdds}/package-info.java | 12 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 55 ++
.../SCMSecurityProtocolClientSideTranslatorPB.java | 132 +++-
.../hdds/scm/metadata/DBTransactionBuffer.java} | 36 +-
.../apache/hadoop/hdds/scm/metadata/Replicate.java | 33 +
.../scm/metadata/SCMDBTransactionBufferImpl.java | 36 +-
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 28 +-
.../hadoop/hdds/scm/metadata}/package-info.java | 13 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 6 +
...lockLocationProtocolClientSideTranslatorPB.java | 46 +-
...inerLocationProtocolClientSideTranslatorPB.java | 123 +++-
.../SCMBlockLocationFailoverProxyProvider.java | 262 +++++++
.../hadoop/hdds/scm/proxy/SCMClientConfig.java | 103 +++
.../SCMContainerLocationFailoverProxyProvider.java | 282 ++++++++
.../apache/hadoop/hdds/scm/proxy/SCMProxyInfo.java | 70 ++
.../SCMSecurityProtocolFailoverProxyProvider.java | 274 ++++++++
.../hadoop/hdds/scm/proxy}/package-info.java | 4 +-
.../hdds/security/x509/certificate/CertInfo.java | 171 +++++
.../certificate/authority/CertificateServer.java | 35 +-
.../certificate/authority/CertificateStore.java | 61 +-
.../certificate/authority/DefaultApprover.java | 16 +-
.../certificate/authority/DefaultCAServer.java | 161 +++--
.../authority/PKIProfiles/DefaultCAProfile.java | 42 +-
.../authority/PKIProfiles/DefaultProfile.java | 12 +-
.../authority/PKIProfiles/PKIProfile.java | 9 +
.../x509/certificate/client/CertificateClient.java | 66 ++
.../certificate/client/DNCertificateClient.java | 5 +
.../client/DefaultCertificateClient.java | 163 ++++-
.../certificate/client/OMCertificateClient.java | 5 +
...ficateClient.java => SCMCertificateClient.java} | 41 +-
.../security/x509/certificate}/package-info.java | 11 +-
.../certificates/utils/CertificateSignRequest.java | 5 +-
.../hadoop/hdds/server/http/ProfileServlet.java | 57 +-
.../hdds/server/http/PrometheusMetricsSink.java | 90 +--
.../apache/hadoop/hdds/utils/DBStoreHAManager.java | 19 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 515 ++++++++++++++
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 106 ++-
.../apache/hadoop/hdds/utils/TransactionInfo.java | 56 +-
.../hadoop/hdds/utils/TransactionInfoCodec.java | 17 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 4 +-
.../hdds/scm/protocol/TestReplicationConfig.java | 66 ++
.../x509/certificate/authority/MockCAStore.java | 37 +-
.../certificate/authority/TestDefaultCAServer.java | 163 ++++-
.../hdds/server/http/TestProfileServlet.java | 9 +-
.../server/http/TestPrometheusMetricsSink.java | 163 +++--
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
.../src/main/proto/ScmAdminProtocol.proto | 21 +-
.../dev-support/findbugsExcludeFile.xml | 3 +
hadoop-hdds/interface-client/pom.xml | 17 +-
.../src/main/proto/DatanodeClientProtocol.proto | 17 +-
.../src/main/proto/InterSCMProtocol.proto | 46 ++
.../interface-client/src/main/proto/hdds.proto | 35 +-
hadoop-hdds/interface-server/pom.xml | 4 +-
.../src/main/proto/SCMRatisProtocol.proto} | 44 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 4 +
.../src/main/proto/ScmServerProtocol.proto | 6 +
.../src/main/proto/ScmServerSecurityProtocol.proto | 61 ++
hadoop-hdds/pom.xml | 4 +-
.../server-scm/dev-support/findbugsExcludeFile.xml | 7 +
hadoop-hdds/server-scm/pom.xml | 9 +-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 110 +++
.../apache/hadoop/hdds/scm/block/BlockManager.java | 5 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 67 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 18 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 33 +-
...lockLogImpl.java => DeletedBlockLogImplV2.java} | 284 ++++----
.../scm/block/DeletedBlockLogStateManager.java | 48 ++
.../scm/block/DeletedBlockLogStateManagerImpl.java | 281 ++++++++
.../hdds/scm/block/SCMBlockDeletingService.java | 87 ++-
.../container/AbstractContainerReportHandler.java | 36 +-
.../scm/container/CloseContainerEventHandler.java | 32 +-
.../scm/container/ContainerActionsHandler.java | 2 +-
.../hdds/scm/container/ContainerManager.java | 19 +
.../hdds/scm/container/ContainerManagerImpl.java | 427 ++++++++++++
...ntainerManager.java => ContainerManagerV2.java} | 132 ++--
.../hdds/scm/container/ContainerReplica.java | 13 +-
.../hdds/scm/container/ContainerReportHandler.java | 21 +-
.../hdds/scm/container/ContainerStateManager.java | 36 +-
.../scm/container/ContainerStateManagerImpl.java | 573 ++++++++++++++++
.../scm/container/ContainerStateManagerV2.java | 189 +++++
.../IncrementalContainerReportHandler.java | 11 +-
.../hdds/scm/container/ReplicationManager.java | 157 ++++-
.../hdds/scm/container/SCMContainerManager.java | 33 +-
.../scm/container/balancer/ContainerBalancer.java | 137 ++++
.../balancer/ContainerBalancerConfiguration.java | 115 ++++
.../hdds/scm/container/balancer/package-info.java | 21 +
.../container/placement/metrics/SCMNodeStat.java | 23 +
.../scm/container/states/ContainerAttribute.java | 2 +-
.../scm/container/states/ContainerStateMap.java | 362 ++++------
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 3 -
.../hadoop/hdds/scm/ha/CheckedConsumer.java} | 55 +-
.../hadoop/hdds/scm/ha/CheckedFunction.java} | 48 +-
.../apache/hadoop/hdds/scm/ha/ExecutionUtil.java | 70 ++
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 378 ++++++++++
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 174 +++++
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 87 +++
.../hadoop/hdds/scm/ha/InterSCMGrpcService.java | 83 +++
.../hdds/scm/ha/MockSCMHADBTransactionBuffer.java | 100 +++
.../hadoop/hdds/scm/ha/MockSCMHAManager.java | 240 +++++++
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 177 +++++
.../apache/hadoop/hdds/scm/ha/ReflectionUtil.java | 67 ++
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 224 ++++++
.../hdds/scm/ha/SCMDBCheckpointProvider.java | 88 +++
.../hadoop/hdds/scm/ha/SCMGrpcOutputStream.java | 131 ++++
.../hdds/scm/ha/SCMHADBTransactionBuffer.java | 44 ++
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 142 ++++
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 123 ++++
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 78 +++
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 363 ++++++++++
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 309 +++++++++
.../apache/hadoop/hdds/scm/ha/SCMNodeDetails.java | 213 ++++++
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 142 ++++
.../hadoop/hdds/scm/ha/SCMRatisResponse.java | 109 +++
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 64 ++
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 343 ++++++++++
.../org/apache/hadoop/hdds/scm/ha/SCMService.java | 75 ++
.../hadoop/hdds/scm/ha/SCMServiceManager.java | 87 +++
.../hadoop/hdds/scm/ha/SCMSnapshotDownloader.java | 43 ++
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 144 ++++
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 311 +++++++++
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 328 +++++++++
.../hadoop/hdds/scm/ha/io/BigIntegerCodec.java} | 52 +-
.../hadoop/hdds/scm/ha/io/BooleanCodec.java} | 50 +-
.../org/apache/hadoop/hdds/scm/ha/io/Codec.java} | 50 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 63 ++
.../apache/hadoop/hdds/scm/ha/io/EnumCodec.java | 50 ++
.../hdds/scm/ha/io/GeneratedMessageCodec.java | 48 ++
.../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 69 ++
.../apache/hadoop/hdds/scm/ha/io/LongCodec.java} | 54 +-
.../apache/hadoop/hdds/scm/ha/io/StringCodec.java} | 50 +-
.../hdds/scm/ha/io/X509CertificateCodec.java | 57 ++
.../hadoop/hdds/scm/ha/io}/package-info.java | 5 +-
.../apache/hadoop/hdds/scm/ha}/package-info.java | 6 +-
.../{ContainerIDCodec.java => CertInfoCodec.java} | 36 +-
.../hadoop/hdds/scm/metadata/ContainerIDCodec.java | 4 +-
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 46 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 77 ++-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 166 +++++
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 8 +-
.../hadoop/hdds/scm/node/NewNodeHandler.java | 30 +-
.../hdds/scm/node/NodeDecommissionManager.java | 40 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 23 +-
.../scm/node/NonHealthyToHealthyNodeHandler.java | 16 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 99 ++-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 2 +-
.../hdds/scm/node/StartDatanodeAdminHandler.java | 2 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 2 +-
.../scm/pipeline/BackgroundPipelineCreatorV2.java | 304 +++++++++
.../hdds/scm/pipeline/PipelineActionHandler.java | 22 +-
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 11 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 19 +-
.../hdds/scm/pipeline/PipelineManagerMXBean.java | 3 +-
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 605 ++++++++++++++++
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 23 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 6 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 17 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 83 ++-
.../scm/pipeline/PipelineStateManagerV2Impl.java | 410 +++++++++++
.../hdds/scm/pipeline/RatisPipelineProvider.java | 23 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 19 +
.../hdds/scm/pipeline/SCMPipelineManager.java | 167 ++---
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
.../hadoop/hdds/scm/pipeline/StateManager.java | 125 ++++
.../algorithms/DefaultLeaderChoosePolicy.java | 4 +-
.../choose/algorithms/LeaderChoosePolicy.java | 8 +-
.../algorithms/LeaderChoosePolicyFactory.java | 6 +-
.../algorithms/MinLeaderCountChoosePolicy.java | 6 +-
.../SCMSecurityProtocolServerSideTranslatorPB.java | 212 +++++-
...lockLocationProtocolServerSideTranslatorPB.java | 31 +
...inerLocationProtocolServerSideTranslatorPB.java | 89 ++-
.../hdds/scm/safemode/SCMSafeModeManager.java | 67 +-
.../scm/server/OzoneStorageContainerManager.java | 6 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 40 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 216 +++++-
.../hdds/scm/server/SCMClientProtocolServer.java | 219 ++++--
.../hadoop/hdds/scm/server/SCMConfigurator.java | 50 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 18 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 181 ++++-
.../hdds/scm/server/SCMStarterInterface.java | 4 +
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 41 ++
.../hdds/scm/server/StorageContainerManager.java | 736 +++++++++++++++-----
.../scm/server/StorageContainerManagerStarter.java | 26 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 25 -
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 2 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 15 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 41 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 86 ++-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 155 ++---
.../hadoop/hdds/scm/container/MockNodeManager.java | 16 +
.../hdds/scm/container/SimpleMockNodeManager.java | 15 +
.../container/TestCloseContainerEventHandler.java | 55 +-
.../scm/container/TestContainerActionsHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 152 +++++
.../scm/container/TestContainerReportHandler.java | 7 +-
.../scm/container/TestContainerStateManager.java | 3 +-
.../TestIncrementalContainerReportHandler.java | 26 +-
.../hdds/scm/container/TestReplicationManager.java | 51 +-
.../scm/container/TestSCMContainerManager.java | 21 +-
.../scm/container/TestUnknownContainerReport.java | 7 +-
.../container/states/TestContainerAttribute.java | 18 +-
.../states/TestContainerReplicaCount.java | 6 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 143 ++++
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 77 +++
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 204 ++++++
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 97 +++
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 89 +++
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 167 +++++
.../hdds/scm/ha/TestSequenceIDGenerator.java | 116 ++++
.../hdds/scm/ha/io/TestBigIntegerCodec.java} | 58 +-
.../hdds/scm/ha/io/TestX509CertificateCodec.java | 61 ++
.../hdds/scm/node/TestContainerPlacement.java | 18 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 2 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 33 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 27 +-
.../hdds/scm/node/TestNodeReportHandler.java | 5 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 4 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 3 +-
.../scm/node/states/TestNode2ContainerMap.java | 10 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 8 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 237 +++++++
.../scm/pipeline/MockRatisPipelineProvider.java | 47 +-
.../scm/pipeline/TestPipelineActionHandler.java | 6 +-
.../TestPipelineDatanodesIntersection.java | 2 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 645 +++++++++++++++++
.../scm/pipeline/TestPipelinePlacementPolicy.java | 22 +-
.../scm/pipeline/TestPipelineStateManager.java | 24 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 27 +-
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 22 +
.../choose/algorithms/TestLeaderChoosePolicy.java | 7 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 78 ++-
.../TestOneReplicaPipelineSafeModeRule.java | 31 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 221 +++---
.../scm/server/TestSCMBlockProtocolServer.java | 6 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 94 ++-
.../scm/server/TestSCMSecurityProtocolServer.java | 5 +-
.../server/TestStorageContainerManagerStarter.java | 35 +
.../ozone/container/common/TestEndPoint.java | 12 +-
.../testutils/ReplicationNodeManagerMock.java | 15 +
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 3 +-
hadoop-hdds/test-utils/pom.xml | 4 +-
hadoop-hdds/tools/pom.xml | 4 +-
.../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 13 +
.../hdds/scm/cli/ContainerOperationClient.java | 117 ++--
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 27 +-
.../hdds/scm/cli/container/ListSubcommand.java | 8 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 13 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 14 +-
.../scm/cli/datanode/RecommissionSubCommand.java | 13 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 81 ++-
.../cli/datanode/TestDecommissionSubCommand.java | 130 ++++
.../cli/datanode/TestMaintenanceSubCommand.java | 135 ++++
.../cli/datanode/TestRecommissionSubCommand.java | 130 ++++
hadoop-ozone/client/pom.xml | 4 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 5 +
.../hadoop/ozone/client/io/KeyOutputStream.java | 2 +-
.../ozone/client/protocol/ClientProtocol.java | 2 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 28 +-
.../hadoop/ozone/client/TestHddsClientUtils.java | 24 +-
hadoop-ozone/common/pom.xml | 4 +-
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 2 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 64 +-
.../org/apache/hadoop/ozone/ha}/package-info.java | 9 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 10 +
.../ozone/om/ha/OMFailoverProxyProvider.java | 3 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 4 +
.../hadoop/ozone/om/helpers/OmOzoneAclMap.java | 357 ----------
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 66 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 2 +-
.../hadoop/ozone/om/helpers/ServiceInfoEx.java | 8 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 2 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 2 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 13 +-
.../hadoop/ozone/om/helpers/TestOmOzoneAclMap.java | 56 --
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 20 +-
.../hadoop/ozone/om/helpers/TestOzoneAclUtil.java | 39 ++
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/datanode/pom.xml | 2 +-
hadoop-ozone/dev-support/checks/_lib.sh | 106 +++
hadoop-ozone/dev-support/checks/acceptance.sh | 5 +
hadoop-ozone/dev-support/checks/bats.sh | 5 +
hadoop-ozone/dev-support/checks/junit.sh | 18 +-
hadoop-ozone/dev-support/checks/kubernetes.sh | 16 +
.../intellij/runConfigurations/OzoneFsShell.xml | 36 +
hadoop-ozone/dist/README.md | 2 +-
.../dist/dev-support/bin/dist-layout-stitching | 3 -
hadoop-ozone/dist/pom.xml | 4 +-
.../dist/src/main/assemblies/ozone-src.xml | 4 +
.../src/main/compose/ozone-ha/docker-compose.yaml | 27 +-
.../dist/src/main/compose/ozone-ha/docker-config | 9 +-
.../dist/src/main/compose/ozone-ha/test.sh | 9 +-
.../dist/src/main/compose/ozone-mr/test.sh | 13 +-
.../{ozonesecure-om-ha => ozonesecure-ha}/.env | 0
.../docker-compose.yaml | 94 ++-
.../docker-config | 13 +-
.../{ozonesecure-om-ha => ozonesecure-ha}/test.sh | 8 +-
.../main/compose/ozonesecure/docker-compose.yaml | 1 +
.../dist/src/main/compose/ozonesecure/test.sh | 51 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 13 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 41 +-
.../compose/upgrade/delete-and-regenerate-data.sh | 53 ++
.../test.sh => upgrade/delete-data.sh} | 23 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 13 +-
.../dist/src/main/dockerlibexec/entrypoint.sh | 5 +
.../k8s/definitions/ozone/definitions/om-ha.yaml | 81 +++
.../k8s/definitions/ozone/definitions/scm-ha.yaml | 81 +++
.../switchtoemptydir.yaml} | 28 +-
.../src/main/k8s/definitions/ozone/flekszible.yaml | 2 +-
.../k8s/definitions/ozone/freon/flekszible.yaml | 2 +-
.../ozone-ha/Flekszible} | 16 +-
.../examples/{ozone => ozone-ha}/LICENSE.header | 0
.../k8s/examples/ozone-ha/config-configmap.yaml | 35 +
.../{ozone => ozone-ha}/datanode-service.yaml | 0
.../{ozone => ozone-ha}/datanode-statefulset.yaml | 0
.../freon/freon-deployment.yaml | 0
.../examples/{ozone => ozone-ha}/om-service.yaml | 0
.../{ozone => ozone-ha}/om-statefulset.yaml | 0
.../examples/{ozone => ozone-ha}/s3g-service.yaml | 0
.../{ozone => ozone-ha}/s3g-statefulset.yaml | 0
.../examples/{ozone => ozone-ha}/scm-service.yaml | 0
.../{ozone => ozone-ha}/scm-statefulset.yaml | 0
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 5 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +-
...-jakarta.validation-jakarta.validation-api.txt} | 233 -------
.../LICENSE-jakarta.ws.rs-jakarta.ws.rs-api.md | 637 +++++++++++++++++
.../LICENSE-javax.ws.rs-javax.ws.rs-api.txt | 759 ---------------------
.../NOTICE-jakarta.ws.rs-jakarta.ws.rs-api.md | 61 ++
hadoop-ozone/dist/src/main/ozone/README.txt | 2 +-
.../src/main/smoketest/admincli/container.robot | 7 +-
.../src/main/smoketest/admincli/pipeline.robot | 3 +-
.../smoketest/admincli/replicationmanager.robot | 5 +-
.../src/main/smoketest/admincli/safemode.robot | 5 +-
.../dist/src/main/smoketest/admincli/scmha.robot | 27 +-
.../dist/src/main/smoketest/basic/basic.robot | 4 +-
.../dist/src/main/smoketest/basic/links.robot | 11 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 24 +-
.../src/main/smoketest/basic/ozone-shell.robot | 3 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 2 +-
.../dist/src/main/smoketest/ozonefs/setup.robot | 2 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 2 +-
.../dist/src/main/smoketest/s3/webui.robot | 2 +-
.../main/smoketest/security/ozone-secure-fs.robot | 19 +-
.../dist/src/main/smoketest/spnego/web.robot | 9 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 5 +
.../dist/src/shell/ozone/ozone-functions.sh | 26 +-
.../dist/src/test/shell/compose_testlib.bats | 6 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 10 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 103 ++-
.../org/apache/hadoop/ozone/OzoneChaosCluster.java | 3 +-
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 2 +-
.../ozone/TestDatanodeMiniChaosOzoneCluster.java | 2 +-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 18 +-
.../TestOzoneManagerMiniChaosOzoneCluster.java | 4 +-
...rageContainerManagerMiniChaosOzoneCluster.java} | 17 +-
.../org/apache/hadoop/ozone/failure/Failures.java | 65 +-
.../fault-injection-test/network-tests/pom.xml | 4 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 4 +-
.../dev-support/findbugsExcludeFile.xml | 8 +
hadoop-ozone/integration-test/pom.xml | 4 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 2 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 4 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 2 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 8 +-
.../fs/ozone/TestOzoneFileSystemMetrics.java | 2 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 13 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 2 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 2 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 159 +++++
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 102 +++
.../TestContainerStateManagerIntegration.java | 19 +-
.../metrics/TestSCMContainerManagerMetrics.java | 23 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 2 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 12 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 18 +-
.../TestRatisPipelineCreateAndDestroy.java | 11 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 8 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 34 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 114 +++-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 485 ++++++++++---
.../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 115 ++++
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 18 +-
.../hadoop/ozone/TestContainerOperations.java | 2 +-
.../TestContainerStateMachineIdempotency.java | 2 +-
.../apache/hadoop/ozone/TestDelegationToken.java | 3 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 2 +-
...ACluster.java => TestMiniOzoneOMHACluster.java} | 10 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 11 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 62 +-
.../hadoop/ozone/TestStorageContainerManager.java | 236 +++++--
.../ozone/TestStorageContainerManagerHelper.java | 2 +-
.../ozone/client/CertificateClientTestImpl.java | 42 ++
.../ozone/client/rpc/Test2WayCommitInRatis.java | 2 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 2 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 2 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 2 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 2 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 2 +-
.../rpc/TestCloseContainerHandlingByClient.java | 2 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 2 +-
.../rpc/TestContainerReplicationEndToEnd.java | 6 +-
.../client/rpc/TestContainerStateMachine.java | 3 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 2 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 19 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 4 +-
.../client/rpc/TestFailureHandlingByClient.java | 12 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 4 +-
.../client/rpc/TestHybridPipelineOnDatanode.java | 2 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 6 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 4 +-
...estOzoneClientRetriesOnExceptionFlushDelay.java | 4 +-
.../rpc/TestOzoneClientRetriesOnExceptions.java | 6 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 6 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 87 ++-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 3 +
.../hadoop/ozone/client/rpc/TestReadRetries.java | 4 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 8 +-
.../client/rpc/read/TestChunkInputStream.java | 207 ++++++
.../ozone/client/rpc/read/TestInputStreamBase.java | 246 +++++++
.../client/rpc/{ => read}/TestKeyInputStream.java | 338 +++------
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 16 +-
.../TestCloseContainerByPipeline.java | 41 +-
.../commandhandler/TestCloseContainerHandler.java | 12 +-
.../commandhandler/TestDeleteContainerHandler.java | 32 +-
.../transport/server/ratis/TestCSMMetrics.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 4 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 5 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 2 +-
.../container/server/TestContainerServer.java | 4 +-
.../server/TestSecureContainerServer.java | 2 +-
.../ozone/dn/TestDatanodeLayoutUpgradeTool.java | 131 ++++
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 +-
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 8 +-
.../freon/TestDataValidateWithDummyContainers.java | 2 +-
.../TestDataValidateWithSafeByteOperations.java | 2 +-
.../TestDataValidateWithUnsafeByteOperations.java | 2 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 2 +-
.../ozone/freon/TestFreonWithDatanodeRestart.java | 3 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 2 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 2 +-
.../ozone/om/TestContainerReportWithKeys.java | 4 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 10 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 2 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 +-
.../hadoop/ozone/om/TestOMEpochForNonRatis.java | 2 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 32 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 2 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 2 +-
.../org/apache/hadoop/ozone/om/TestOmInit.java | 2 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 +-
.../ozone/om/TestOzoneManagerConfiguration.java | 23 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 12 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 16 +-
.../ozone/om/TestOzoneManagerListVolumes.java | 2 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 2 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 2 +-
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 2 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 6 +-
.../ozone/om/parser/TestOMRatisLogParser.java | 8 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 14 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 46 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 22 +-
.../ozone/recon/TestReconWithOzoneManager.java | 2 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 10 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 2 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 2 +-
.../hadoop/ozone/scm/TestContainerSmallFile.java | 8 +-
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 2 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 2 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 358 ++++++++++
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 18 +-
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 2 +-
.../ozone/scm/TestStorageContainerManagerHA.java | 250 +++++++
.../hadoop/ozone/scm/TestXceiverClientManager.java | 12 +-
.../hadoop/ozone/scm/TestXceiverClientMetrics.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../scm/pipeline/TestPipelineManagerMXBean.java | 2 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 2 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 5 +-
.../hadoop/ozone/shell/TestOzoneDatanodeShell.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 44 +-
.../apache/hadoop/ozone/shell/TestScmAdminHA.java | 80 +++
hadoop-ozone/interface-client/pom.xml | 4 +-
.../src/main/proto/OmClientProtocol.proto | 2 +
hadoop-ozone/interface-storage/pom.xml | 4 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 7 +-
...nfoCodec.java => TestTransactionInfoCodec.java} | 21 +-
.../ozone/om/helpers/TestInstanceHelper.java | 68 ++
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 65 +-
hadoop-ozone/ozone-manager/pom.xml | 4 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 8 +-
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 18 -
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 14 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 177 ++---
.../hadoop/ozone/om/OzoneManagerStarter.java | 4 +
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 69 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 15 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 60 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 9 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 40 +-
.../apache/hadoop/ozone/om/ha/OMNodeDetails.java | 94 +--
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 5 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 14 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 18 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 80 +--
.../om/request/bucket/OMBucketCreateRequest.java | 11 +-
.../key/acl/prefix/OMPrefixAddAclRequest.java | 1 -
.../S3InitiateMultipartUploadRequest.java | 7 +-
.../multipart/S3MultipartUploadAbortRequest.java | 7 +-
.../S3MultipartUploadCommitPartRequest.java | 7 +-
.../S3MultipartUploadCompleteRequest.java | 6 +-
.../om/request/volume/OMVolumeCreateRequest.java | 3 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 5 +
.../ozone/om/ScmBlockLocationTestingClient.java | 6 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 14 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 9 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 9 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 6 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 3 +-
.../volume/acl/TestOMVolumeAddAclRequest.java | 10 +-
.../volume/acl/TestOMVolumeRemoveAclRequest.java | 12 +-
.../volume/acl/TestOMVolumeSetAclRequest.java | 16 +-
hadoop-ozone/ozonefs-common/pom.xml | 2 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 10 +
hadoop-ozone/ozonefs-hadoop2/pom.xml | 2 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 2 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 2 +-
hadoop-ozone/ozonefs/pom.xml | 2 +-
hadoop-ozone/pom.xml | 4 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
hadoop-ozone/recon/pom.xml | 17 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 6 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 2 +
.../ozone/recon/api/ClusterStateEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 4 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 8 +-
.../ozone/recon/scm/ReconContainerManager.java | 63 +-
.../recon/scm/ReconContainerReportHandler.java | 6 +-
.../recon/scm/ReconDatanodeProtocolServer.java | 6 +-
.../ReconIncrementalContainerReportHandler.java | 12 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 4 +-
.../ozone/recon/scm/ReconPipelineManager.java | 97 +--
.../recon/scm/ReconPipelineReportHandler.java | 4 +-
.../scm/ReconStorageContainerManagerFacade.java | 60 +-
.../resources/webapps/recon/ozone-recon-web/NOTICE | 2 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 66 +-
.../ozone/recon/api/TestContainerEndpoint.java | 22 +-
.../recon/fsck/TestContainerHealthStatus.java | 2 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 28 +-
.../TestContainerHealthTaskRecordGenerator.java | 2 +-
.../scm/AbstractReconContainerManagerTest.java | 38 +-
.../ozone/recon/scm/TestReconContainerManager.java | 50 +-
...TestReconIncrementalContainerReportHandler.java | 15 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 48 +-
.../recon/scm/TestReconPipelineReportHandler.java | 4 +-
hadoop-ozone/s3gateway/pom.xml | 6 +-
.../hadoop/ozone/s3/AWSSignatureProcessor.java | 462 -------------
.../hadoop/ozone/s3/EmptyContentTypeFilter.java | 131 ++++
.../hadoop/ozone/s3/OzoneClientProducer.java | 92 ++-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 14 -
.../hadoop/ozone/s3/exception/OS3Exception.java | 14 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 2 +-
.../ozone/s3/signature/AWSSignatureProcessor.java | 205 ++++++
.../AuthorizationV2HeaderParser.java} | 70 +-
.../AuthorizationV4HeaderParser.java} | 206 +++---
.../s3/signature/AuthorizationV4QueryParser.java | 98 +++
.../ozone/s3/{header => signature}/Credential.java | 8 +-
.../hadoop/ozone/s3/signature/SignatureInfo.java | 113 +++
.../hadoop/ozone/s3/signature/SignatureParser.java | 35 +-
.../s3/{ => signature}/SignatureProcessor.java | 30 +-
.../ozone/s3/signature/StringToSignProducer.java | 324 +++++++++
.../s3/{header => signature}/package-info.java | 2 +-
.../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 3 +-
.../resources/webapps/s3gateway/WEB-INF/web.xml | 9 +
.../src/main/resources/webapps/static/index.html | 6 +-
.../hadoop/ozone/s3/TestAWSSignatureProcessor.java | 141 ----
.../ozone/s3/TestEmptyContentTypeFilter.java | 58 ++
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 100 +--
.../hadoop/ozone/s3/endpoint/TestBucketPut.java | 31 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 24 -
.../TestAuthorizationV2HeaderParser.java} | 35 +-
.../TestAuthorizationV4HeaderParser.java} | 153 +++--
.../signature/TestAuthorizationV4QueryParser.java | 146 ++++
.../s3/signature/TestStringToSignProducer.java | 90 +++
.../hadoop/ozone/s3/util/TestOzoneS3Util.java | 18 +-
hadoop-ozone/tools/pom.xml | 4 +-
.../admin/scm/GetScmRatisRolesSubcommand.java | 45 ++
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 64 ++
.../hadoop/ozone/admin/scm}/package-info.java | 12 +-
.../apache/hadoop/ozone/debug/DatanodeLayout.java | 106 +++
.../apache/hadoop/ozone/debug/ExportContainer.java | 6 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 38 +-
.../ozone/freon/ClosedContainerReplicator.java | 2 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 22 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 6 +
.../freon/containergenerator/BaseGenerator.java | 80 +++
.../containergenerator/GeneratorDatanode.java | 329 +++++++++
.../freon/containergenerator/GeneratorOm.java | 280 ++++++++
.../freon/containergenerator/GeneratorScm.java | 92 +++
.../freon/containergenerator}/package-info.java | 10 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 5 +-
.../ozone/genesis/BenchmarkChunkManager.java | 2 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 3 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 25 +-
.../ozone/shell/prefix/AddAclPrefixHandler.java | 52 ++
.../ozone/shell/prefix/GetAclPrefixHandler.java | 40 ++
.../hadoop/ozone/shell/prefix/PrefixCommands.java | 75 ++
.../hadoop/ozone/shell/prefix/PrefixUri.java | 48 ++
.../ozone/shell/prefix/RemoveAclPrefixHandler.java | 38 +-
.../ozone/shell/prefix/SetAclPrefixHandler.java | 53 ++
.../hadoop/ozone/shell/prefix/package-info.java | 21 +
.../hadoop/ozone/shell/TestOzoneAddress.java | 11 +
pom.xml | 32 +-
767 files changed, 31744 insertions(+), 7880 deletions(-)
copy .github/workflows/{close-pending.yaml => cancel-ci.yaml} (60%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/RatisReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StandaloneReplicationConfig.java
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/{ScmInfo.java => AddSCMRequest.java} (50%)
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/{ozone/audit/SCMAction.java => hdds/scm/DatanodeAdminError.java} (52%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/package-info.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/RetriableWithNoFailoverException.java (77%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha}/package-info.java (88%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/utils/ClientCommandsUtils.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/ratis => common/src/main/java/org/apache/hadoop/hdds/scm/utils}/package-info.java (87%)
rename hadoop-hdds/{framework => common}/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java (100%)
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header => hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server}/package-info.java (88%)
rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OMRatisSnapshotInfo.java => hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ha/ratis/RatisSnapshotInfo.java (84%)
copy hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/{audit/SCMAction.java => common/ha/ratis/package-info.java} (52%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/utils/BufferUtils.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/ratis => common/src/main/java/org/apache/hadoop/ozone/common/utils}/package-info.java (87%)
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/ConfUtils.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ha/package-info.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header => hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/ha}/package-info.java (89%)
rename hadoop-hdds/docs/content/feature/{HA.md => OM-HA.md} (78%)
rename hadoop-hdds/docs/content/feature/{HA.zh.md => OM-HA.zh.md} (83%)
create mode 100644 hadoop-hdds/docs/content/feature/SCM-HA.md
create mode 100644 hadoop-hdds/docs/content/feature/SCM-HA.zh.md
rename {hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds}/ExitManager.java (96%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/NodeDetails.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/ratis => framework/src/main/java/org/apache/hadoop/hdds}/package-info.java (78%)
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java => framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/DBTransactionBuffer.java} (52%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/Replicate.java
copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfoEx.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBTransactionBufferImpl.java (56%)
rename hadoop-hdds/{server-scm => framework}/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java (83%)
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata}/package-info.java (62%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMClientConfig.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMProxyInfo.java
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMSecurityProtocolFailoverProxyProvider.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/ratis => framework/src/main/java/org/apache/hadoop/hdds/scm/proxy}/package-info.java (88%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/CertInfo.java
copy hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/{OMCertificateClient.java => SCMCertificateClient.java} (78%)
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate}/package-info.java (78%)
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/package-info.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBStoreHAManager.java (67%)
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
rename hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/ratis/OMTransactionInfo.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java (73%)
rename hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OMTransactionInfoCodec.java => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfoCodec.java (69%)
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocol/TestReplicationConfig.java
create mode 100644 hadoop-hdds/interface-client/src/main/proto/InterSCMProtocol.proto
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerMXBean.java => interface-server/src/main/proto/SCMRatisProtocol.proto} (52%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/{DeletedBlockLogImpl.java => DeletedBlockLogImplV2.java} (61%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ContainerManager.java => ContainerManagerV2.java} (66%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerV2.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/package-info.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/CheckedConsumer.java} (52%)
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/CheckedFunction.java} (52%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/ExecutionUtil.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcService.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockSCMHADBTransactionBuffer.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/MockSCMHAManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/ReflectionUtil.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMContext.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMDBCheckpointProvider.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMGrpcOutputStream.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHADBTransactionBuffer.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHADBTransactionBufferImpl.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeDetails.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisResponse.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServer.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMService.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMServiceManager.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotDownloader.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BigIntegerCodec.java} (52%)
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BooleanCodec.java} (51%)
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/Codec.java} (50%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/EnumCodec.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ListCodec.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/LongCodec.java} (51%)
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/StringCodec.java} (51%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/X509CertificateCodec.java
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io}/package-info.java (88%)
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header => hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha}/package-info.java (88%)
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/{ContainerIDCodec.java => CertInfoCodec.java} (50%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeUsageInfo.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManagerV2Impl.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/StateManager.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestReplicationAnnotation.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMContext.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMServiceManager.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIDGenerator.java
copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java => server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java} (51%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestX509CertificateCodec.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDecommissionSubCommand.java
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestMaintenanceSubCommand.java
create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestRecommissionSubCommand.java
copy hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header => common/src/main/java/org/apache/hadoop/ozone/ha}/package-info.java (88%)
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmOzoneAclMap.java
delete mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmOzoneAclMap.java
create mode 100644 hadoop-ozone/dev-support/checks/_lib.sh
create mode 100644 hadoop-ozone/dev-support/intellij/runConfigurations/OzoneFsShell.xml
rename hadoop-ozone/dist/src/main/compose/{ozonesecure-om-ha => ozonesecure-ha}/.env (100%)
rename hadoop-ozone/dist/src/main/compose/{ozonesecure-om-ha => ozonesecure-ha}/docker-compose.yaml (69%)
rename hadoop-ozone/dist/src/main/compose/{ozonesecure-om-ha => ozonesecure-ha}/docker-config (92%)
copy hadoop-ozone/dist/src/main/compose/{ozonesecure-om-ha => ozonesecure-ha}/test.sh (86%)
create mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/delete-and-regenerate-data.sh
rename hadoop-ozone/dist/src/main/compose/{ozonesecure-om-ha/test.sh => upgrade/delete-data.sh} (70%)
create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/om-ha.yaml
create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/scm-ha.yaml
copy hadoop-ozone/dist/src/main/k8s/definitions/ozone/{flekszible.yaml => definitions/switchtoemptydir.yaml} (62%)
copy hadoop-ozone/dist/src/main/k8s/{definitions/ozone/freon/flekszible.yaml => examples/ozone-ha/Flekszible} (71%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/LICENSE.header (100%)
create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/datanode-service.yaml (100%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/datanode-statefulset.yaml (100%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/freon/freon-deployment.yaml (100%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/om-service.yaml (100%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/om-statefulset.yaml (100%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/s3g-service.yaml (100%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/s3g-statefulset.yaml (100%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/scm-service.yaml (100%)
copy hadoop-ozone/dist/src/main/k8s/examples/{ozone => ozone-ha}/scm-statefulset.yaml (100%)
copy hadoop-ozone/dist/src/main/license/bin/{LICENSE.txt => licenses/LICENSE-jakarta.validation-jakarta.validation-api.txt} (62%)
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-jakarta.ws.rs-jakarta.ws.rs-api.md
delete mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.ws.rs-javax.ws.rs-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/NOTICE-jakarta.ws.rs-jakarta.ws.rs-api.md
copy .github/workflows/comments.yaml => hadoop-ozone/dist/src/main/smoketest/admincli/scmha.robot (66%)
copy hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/{TestOzoneManagerMiniChaosOzoneCluster.java => TestStorageContainerManagerMiniChaosOzoneCluster.java} (80%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMHAClusterImpl.java
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{TestMiniOzoneHACluster.java => TestMiniOzoneOMHACluster.java} (92%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java
rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/{ => read}/TestKeyInputStream.java (51%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeLayoutUpgradeTool.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestScmAdminHA.java
rename hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/codec/{TestOMTransactionInfoCodec.java => TestTransactionInfoCodec.java} (78%)
create mode 100644 hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestInstanceHelper.java
delete mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/AWSSignatureProcessor.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EmptyContentTypeFilter.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java
rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{header/AuthorizationHeaderV2.java => signature/AuthorizationV2HeaderParser.java} (66%)
rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{header/AuthorizationHeaderV4.java => signature/AuthorizationV4HeaderParser.java} (57%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java
rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{header => signature}/Credential.java (93%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java
copy hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java => hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java (67%)
rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{ => signature}/SignatureProcessor.java (63%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
copy hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{header => signature}/package-info.java (95%)
delete mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestAWSSignatureProcessor.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestEmptyContentTypeFilter.java
rename hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/{header/TestAuthorizationHeaderV2.java => signature/TestAuthorizationV2HeaderParser.java} (69%)
rename hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/{header/TestAuthorizationHeaderV4.java => signature/TestAuthorizationV4HeaderParser.java} (71%)
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/GetScmRatisRolesSubcommand.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/ScmAdmin.java
rename hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header => tools/src/main/java/org/apache/hadoop/ozone/admin/scm}/package-info.java (82%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DatanodeLayout.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/BaseGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorOm.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorScm.java
rename {hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ratis => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator}/package-info.java (82%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/prefix/AddAclPrefixHandler.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/prefix/GetAclPrefixHandler.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/prefix/PrefixCommands.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/prefix/PrefixUri.java
copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerIDCodec.java => hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/prefix/RemoveAclPrefixHandler.java (51%)
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/prefix/SetAclPrefixHandler.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/prefix/package-info.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org