You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by na...@apache.org on 2022/07/07 06:23:50 UTC
[ozone] branch HDDS-6517-Snapshot updated (18ea216615 -> 243efe25d3)
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a change to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 18ea216615 HDDS-6839. Migrate tests in hdds-client to JUnit5 (#3490)
add 34eb378399 HDDS-6846. HeadOp ignored for link bucket (#3495)
add 4f0bd4ae3a HDDS-6845. Add CI check for pull request title (#3494)
add 3d623a8dd3 HDDS-6837. Avoid duplicate headers in HTTP response (#3500)
add b0b3f0d25b HDDS-6869 Use UGI when checking Upgrade priv (#3502)
add b1fc69e059 HDDS-6874. Migrate simple tests in hdds-common to JUnit5 (#3507)
add 11fe8d93d0 HDDS-6878. Only export span if Ozone tracer init (#3509)
add 126fa82fa1 HDDS-6799. Refactor OMLockMetrics code in OzoneManagerLock (#3454)
add 47eddeb371 HDDS-6260. EC: Standalone containers should not move to quasi-closed (#3439)
add 4b425fd6cd HDDS-6830. EC: SCMContainerPlacementRackScatter may choose fewer nodes than required (#3511)
add e5318a38a9 HDDS-6841. EC: Validate the server default configuration on Ozone manager startup (#3498)
add be29c6761f HDDS-6697. EC: ReplicationManager - create class to detect EC container health issues (#3512)
add 45f13fd044 HDDS-6840. Add MetaData volume information to the SCM and OM - UI (#3488)
add e525a48768 HDDS-6872. TestAuthorizationV4QueryParser should pass offline (#3506)
add 17d3301a90 HDDS-6870 Clean up isTenantAdmin to use UGI (#3503)
add e4ed9d436c HDDS-6577. Configurations to reserve HDDS volume space. (#3484)
add 8665dc2dc7 HDDS-6898. [SCM HA finalization] Modify acceptance test configuration to speed up test finalization (#3521)
add 94945aed40 HDDS-6829. Limit the no of inflight replication tasks in SCM. (#3482)
add f57a0193c1 HDDS-6806. EC: Implement the EC Reconstruction coordinator. (#3504)
add 3768bc6f0c HDDS-6752. Migrate tests with rules in hdds-server-scm to JUnit5 (#3442)
add 54b38c995e HDDS-6902. Duplicate mockito-core entries in pom.xml (#3525)
add 5d6ce84390 HDDS-6890. EC: Fix potential wrong replica read with over-replicated container. (#3523)
add d86d6a81bc HDDS-6882. Correct exit code for invalid arguments passed to command-line tools. (#3517)
add 321f5856e1 HDDS-4123. Integrate OM Open Key Cleanup Service Into Existing Code (#3319)
add 50dbf08211 HDDS-6695. Enable SCM Ratis by default for new clusters only (#3499)
add bf71b54e2d HDDS-6899. [EC] Remove warnings and errors from console during online reconstruction of data. (#3522)
add 4b20cd6ea1 HDDS-6924. OBJECT_STORE isn't flat namespaced (#3533)
add 88287edc3a HDDS-6875. Migrate parameterized tests in hdds-common to JUnit5 (#3513)
add 67ecb07864 HDDS-6907. OFS should create buckets with FILE_SYSTEM_OPTIMIZED layout. (#3528)
add 349f47d8a3 HDDS-6877. Keep replication port unchanged when restarting datanode in MiniOzoneCluster (#3510)
add c634a871fa HDDS-6868 Add S3Auth information to thread local (#3527)
add 842177d7d0 HDDS-6932. [Multi-Tenant] Move Ranger plugin version to a separate tag (#3537)
add a1d5d8881f HDDS-6876. Migrate test with rules in hdds-common to JUnit5 (#3536)
add 8d2b675cc8 HDDS-6891. Add CapacityVolumeChoosingPolicy (#3519)
add 11975a5b0a HDDS-6936. Mark TestContainerStateMachineFailures#testApplyTransactionFailure as flaky (#3538)
add 9f43b0efbf HDDS-6873. EC: Add block token support for ECReconstructionCoordinator (#3539)
add 2ffbfff0c1 HDDS-5916. Datanodes stuck in leader election in Kubernetes (#3186)
add 110dca5ec7 HDDS-6812. Fix INTERNAL_ERROR message on failed write. (#3529)
add a19414a2bb HDDS-6835. EC: Provide correct example for EC in ozone.server.default.replication (#3524)
add 24a046c224 HDDS-6941. Setting Bucket Property can corrupt bucket layout. (#3543)
add 9367c7f25d HDDS-6906. Fix kerberos config for secure smoke tests. (#3540)
add dbcd2c93a7 HDDS-6937. Close Rocks objects properly in TrashOzoneFilesystem (#3541)
add 6354406b4a HDDS-6944. EC: Handle reconstructECContainersCommand in heartbeat (#3548)
add 178103b9c8 HDDS-6943. Reject property change requests on link buckets (#3546)
add 5cf5c97aee HDDS-6699. EC: ReplicationManager - collect under and over replicated containers (#3545)
add 897b09a567 HDDS-6947. Bump gson version to 2.9.0. (#3558)
add a8808d1c37 HDDS-6949. Audit bucket ACL changes in S3 Gateway. (#3552)
add 965d31cde9 HDDS-6945. EC: EC Reconstruction Command count queues should be included in DN heartbeat (#3550)
add 576a3ebc0a HDDS-6934. OM crashed with OzoneManagerDoubleBuffer (#3544)
add f7bd665bac HDDS-6948. MultiTenantAccessAuthorizerRangerPlugin#deletePolicyByName should log and exit if getAccessPolicyByName returns null (#3551)
add 1245c7cfcd HDDS-6928. ozone container balancer CLI went in hung state due to deadlock (#3535)
add 7a1a626758 HDDS-6968. [Multi-Tenant] Fix USER_MISMATCH error even on correct user. (#3566)
add 6537056b22 HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode (#3542)
add b76ca56968 HDDS-6973. Refactor ObjectEndpoint. (#3569)
add 7ade19e2c3 HDDS-6974. Container report processing in Recon is single threaded. (#3571)
add 38c2ff35f4 HDDS-6879. Bucket layout as null on default layout type (#3561)
add 7838582393 HDDS-6959. Fix move timeout in latest iteration metric (#3562)
add ba4352bafe HDDS-6676. KeyValueContainerData#getProtoBufMessage() should set block count (#3371)
add de77e7a432 HDDS-6773. Cleanup TestRDBTableStore (#3434)
add 54adfd7c27 HDDS-6773. Cleanup TestRDBTableStore (#3434) - fix checkstyle
add 5dd64f37e9 HDDS-6217. Cleanup XceiverClientGrpc TODOs, and document how the client works and should be used. (#3012)
add c02806fd55 HDDS-6977. EC: Remove references to ContainerReplicaPendingOps in TestECContainerReplicaCount (#3575)
add a6500f6124 HDDS-6970. EC: Ensure DatanodeAdminMonitor can handle EC containers during decommission (#3573)
add 30f45b0da5 HDDS-6969. Add link to compose directory in smoketest README (#3567)
add 5e81be16c2 HDDS-6965. Increase timeout for basic check (#3563)
add c5e374544c HDDS-6942. Ozone Buckets/Objects created via S3 should not allow group access (#3553)
add 179755eab0 HDDS-6723. Close Rocks objects properly in OzoneManager (#3400)
add 03cd7c4c5f HDDS-6957. EC: ReplicationManager - priortise under replicated containers (#3574)
add 3873753f23 HDDS-6979. Remove unused plexus dependency declaration (#3579)
add fac23c9f3a HDDS-6960. EC: Implement the Over-replication Handler (#3572)
add 243efe25d3 HDDS-6938. handle NPE when removing prefixAcl (#3568)
No new revisions were added by this update.
Summary of changes:
.github/workflows/post-commit.yml | 2 +-
.../workflows/pull-request.yml | 31 +-
dev-support/ci/pr_title_check.bats | 107 +++++
dev-support/ci/pr_title_check.sh | 48 ++
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 18 +
.../java/org/apache/hadoop/hdds/scm/README.gRPC.md | 252 ++++++++++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 105 ++---
.../hadoop/hdds/scm/XceiverClientManager.java | 33 ++
.../org/apache/hadoop/hdds/scm/package-info.java | 20 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 5 +
.../hadoop/ozone/client/io/ECBlockInputStream.java | 34 +-
.../ozone/client/io/ECBlockInputStreamProxy.java | 11 +-
.../ozone/client/io/TestECBlockInputStream.java | 37 ++
hadoop-hdds/common/pom.xml | 5 +
.../org/apache/hadoop/hdds/cli/GenericCli.java | 24 +-
.../hadoop/hdds/client/ECReplicationConfig.java | 5 +
.../hadoop/hdds/client/RatisReplicationConfig.java | 5 +
.../hadoop/hdds/client/ReplicationConfig.java | 5 +-
.../hdds/client/ReplicationConfigValidator.java | 14 +-
.../hdds/client/StandaloneReplicationConfig.java | 5 +
.../hadoop/hdds/conf/DefaultConfigManager.java | 54 +++
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 8 +
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 23 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 4 +-
.../hadoop/hdds/scm/net/NetworkTopology.java | 8 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 71 ++-
.../hdds/scm/storage/ContainerProtocolCalls.java | 19 +-
.../apache/hadoop/hdds/tracing/TracingUtil.java | 5 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 9 +
.../common/src/main/resources/ozone-default.xml | 45 +-
.../hdds/TestComponentVersionInvariants.java | 76 ++--
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 67 ++-
.../hdds/client/TestECReplicationConfig.java | 32 +-
.../hadoop/hdds/client/TestReplicationConfig.java | 242 +++++-----
.../client/TestReplicationConfigValidator.java | 24 +-
.../conf/TestGeneratedConfigurationOverwrite.java | 18 +-
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 95 ++--
.../hdds/fs/TestCachingSpaceUsageSource.java | 2 +-
.../java/org/apache/hadoop/hdds/fs/TestDU.java | 10 +-
.../org/apache/hadoop/hdds/fs/TestDUFactory.java | 2 +-
.../hdds/fs/TestDedicatedDiskSpaceUsage.java | 12 +-
.../fs/TestDedicatedDiskSpaceUsageFactory.java | 2 +-
.../hadoop/hdds/fs/TestSaveSpaceUsageToFile.java | 16 +-
.../hadoop/hdds/fs/TestSpaceUsageFactory.java | 6 +-
.../hadoop/hdds/protocol/TestDatanodeDetails.java | 8 +-
.../ratis/TestContainerCommandRequestMessage.java | 6 +-
.../apache/hadoop/hdds/ratis/TestRatisHelper.java | 38 +-
.../hdds/ratis/TestServerNotLeaderException.java | 16 +-
.../hdds/scm/container/TestContainerInfo.java | 30 +-
.../scm/container/TestContainerReplicaInfo.java | 36 +-
.../container/TestReplicationManagerReport.java | 102 +++--
.../container/common/helpers/TestExcludeList.java | 14 +-
.../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 51 ++-
.../apache/hadoop/hdds/scm/net/TestNetUtils.java | 2 +-
.../hdds/scm/net/TestNetworkTopologyImpl.java | 281 +++++++-----
.../hadoop/hdds/scm/net/TestNodeSchemaLoader.java | 129 +++---
.../hadoop/hdds/scm/net/TestNodeSchemaManager.java | 40 +-
.../hadoop/hdds/scm/net/TestYamlSchemaLoader.java | 95 ++--
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 42 +-
.../hadoop/hdds/tracing/TestTraceAllMethod.java | 6 +-
.../hadoop/hdds/tracing/TestTracingUtil.java | 23 +-
.../hdds/upgrade/TestHDDSLayoutVersionManager.java | 14 +-
.../hadoop/hdds/utils/TestHddsIdFactory.java | 12 +-
.../hadoop/hdds/utils/TestResourceLimitCache.java | 33 +-
.../hadoop/hdds/utils/TestResourceSemaphore.java | 28 +-
.../hadoop/hdds/utils/TestRetriableTask.java | 4 +-
.../org/apache/hadoop/ozone/TestOzoneConsts.java | 6 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 24 +-
.../apache/hadoop/ozone/common/TestChecksum.java | 19 +-
.../ozone/common/TestChecksumByteBuffer.java | 6 +-
.../common/TestChecksumImplsComputeSameValues.java | 4 +-
.../hadoop/ozone/common/TestChunkBuffer.java | 58 +--
.../TestChunkBufferImplWithByteBufferList.java | 2 +-
.../hadoop/ozone/common/TestStateMachine.java | 56 +--
.../apache/hadoop/ozone/ha/TestOzoneNetUtils.java | 8 +-
.../hadoop/ozone/lease/TestLeaseManager.java | 153 +++----
.../apache/hadoop/ozone/lock/TestLockManager.java | 37 +-
.../upgrade/TestAbstractLayoutVersionManager.java | 44 +-
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 8 +-
.../TestDefaultUpgradeFinalizationExecutor.java | 2 +-
.../upgrade/TestLayoutVersionInstanceFactory.java | 6 +-
.../ozone/upgrade/TestUpgradeFinalizerActions.java | 17 +-
.../ozone/container/common/impl/ContainerData.java | 7 +-
.../common/statemachine/DatanodeStateMachine.java | 10 +-
.../CloseContainerCommandHandler.java | 9 +
.../ClosePipelineCommandHandler.java | 5 +
.../commandhandler/CommandDispatcher.java | 7 +-
.../commandhandler/CommandHandler.java | 8 +-
.../CreatePipelineCommandHandler.java | 5 +
.../FinalizeNewLayoutVersionCommandHandler.java | 5 +
.../ReconstructECContainersCommandHandler.java | 9 +-
.../RefreshVolumeUsageCommandHandler.java | 5 +
.../SetNodeOperationalStateCommandHandler.java | 5 +
.../common/states/datanode/InitDatanodeState.java | 2 +-
.../states/endpoint/HeartbeatEndpointTask.java | 110 ++---
.../volume/CapacityVolumeChoosingPolicy.java | 93 ++++
.../container/common/volume/StorageVolume.java | 12 +
.../ozone/container/common/volume/VolumeInfo.java | 8 +
.../ozone/container/common/volume/VolumeUsage.java | 8 +
.../reconstruction/ECContainerOperationClient.java | 148 ++++++
.../ECReconstructionCoordinator.java | 417 +++++++++++++++++
.../ECReconstructionCoordinatorTask.java | 48 +-
.../reconstruction/ECReconstructionSupervisor.java | 48 +-
.../container/ec/reconstruction/TokenHelper.java | 125 +++++
.../container/keyvalue/KeyValueContainerData.java | 1 +
.../ozone/container/keyvalue/KeyValueHandler.java | 1 +
.../background/BlockDeletingService.java | 4 +-
.../container/metadata/AbstractDatanodeStore.java | 8 +
.../commands/ReconstructECContainersCommand.java | 2 +-
.../common/TestKeyValueContainerData.java | 4 +
.../TestCloseContainerCommandHandler.java | 17 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 42 ++
.../volume/TestCapacityVolumeChoosingPolicy.java | 134 ++++++
.../TestECReconstructionSupervisor.java | 58 ++-
.../container/keyvalue/TestKeyValueContainer.java | 27 ++
.../token}/OzoneBlockTokenSecretManager.java | 4 +-
.../hadoop/hdds/server/events/EventQueue.java | 24 +-
.../hadoop/hdds/server/http/HttpServer2.java | 2 +-
.../hadoop/hdds/server/http/NoCacheFilter.java | 6 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 2 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 4 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 45 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 20 +-
.../token}/TestOzoneBlockTokenSecretManager.java | 6 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 78 +---
.../scm/container/CloseContainerEventHandler.java | 12 +-
...aCount.java => RatisContainerReplicaCount.java} | 60 +--
.../scm/container/balancer/ContainerBalancer.java | 141 +++---
.../balancer/ContainerBalancerMetrics.java | 37 ++
.../SCMContainerPlacementRackScatter.java | 15 +-
.../AbstractOverReplicationHandler.java | 110 +++++
.../replication/ContainerHealthCheck.java} | 40 +-
.../replication/ContainerHealthResult.java | 229 ++++++++++
.../replication/ContainerReplicaCount.java | 69 +++
.../container/replication/ContainerReplicaOp.java | 6 +
.../replication/ECContainerHealthCheck.java | 86 ++++
.../{ => replication}/ECContainerReplicaCount.java | 190 ++++++--
.../replication/ECOverReplicationHandler.java | 165 +++++++
.../replication/ECUnderReplicationHandler.java | 239 ++++++++++
.../scm/container/replication/InflightType.java} | 18 +-
.../replication/LegacyReplicationManager.java | 333 ++++++++++----
.../container/replication/ReplicationManager.java | 198 ++++++--
.../replication/ReplicationManagerMetrics.java | 48 +-
.../replication/UnhealthyReplicationHandler.java | 49 ++
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 7 +
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 59 ++-
.../org/apache/hadoop/hdds/scm/ha/SCMService.java | 1 +
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 6 +-
.../hadoop/hdds/scm/node/NewNodeHandler.java | 1 +
.../hdds/scm/node/NodeAddressUpdateHandler.java | 69 +++
.../hadoop/hdds/scm/node/NodeStateManager.java | 22 +
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 9 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 105 ++++-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 25 +
.../scm/pipeline/BackgroundPipelineCreator.java | 9 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 15 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 91 +++-
.../scm/pipeline/WritableECContainerProvider.java | 2 +-
.../pipeline/WritableRatisContainerProvider.java | 62 ++-
...lockLocationProtocolServerSideTranslatorPB.java | 2 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 8 +
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 3 +
.../hdds/scm/server/StorageContainerManager.java | 47 +-
.../main/resources/webapps/scm/scm-overview.html | 14 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 73 ++-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 22 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 131 +++---
.../container/TestCloseContainerEventHandler.java | 318 ++++++-------
.../container/balancer/TestContainerBalancer.java | 133 ++++--
.../container/replication/ReplicationTestUtil.java | 145 ++++++
.../TestContainerReplicaPendingOps.java | 90 ++--
.../replication/TestECContainerHealthCheck.java | 221 +++++++++
.../replication/TestECContainerReplicaCount.java | 505 +++++++++++++++++++++
.../replication/TestECOverReplicationHandler.java | 159 +++++++
.../replication/TestECUnderReplicationHandler.java | 190 ++++++++
.../TestLegacyReplicationManager.java} | 163 +++++--
.../TestRatisContainerReplicaCount.java} | 261 +++++------
.../replication/TestReplicationManager.java | 339 ++++++++++++++
.../replication/TestReplicationManagerMetrics.java | 6 +
.../container/states/TestContainerAttribute.java | 47 +-
.../states/TestECContainerReplicaCount.java | 432 ------------------
.../hdds/scm/crl/TestCRLStatusReportHandler.java | 40 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 70 ++-
.../hdds/scm/node/TestContainerPlacement.java | 21 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 5 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 38 ++
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 249 ++++++----
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 98 ++--
.../scm/node/states/TestNode2ContainerMap.java | 96 ++--
.../hdds/scm/pipeline/MockPipelineManager.java | 5 +
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 194 ++++++++
.../scm/pipeline/TestPipelinePlacementPolicy.java | 162 +++----
.../pipeline/TestWritableECContainerProvider.java | 32 +-
.../TestOneReplicaPipelineSafeModeRule.java | 20 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 142 +++---
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 49 +-
.../scm/server/TestSCMSecurityProtocolServer.java | 17 +-
.../server/TestStorageContainerManagerStarter.java | 53 +--
.../hdds/scm/update/server/MockCRLStore.java | 6 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 80 ++--
.../TestSCMHAUnfinalizedStateValidationAction.java | 14 +-
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 33 +-
.../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 9 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 10 +-
hadoop-ozone/common/pom.xml | 9 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 5 +
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 1 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 121 +++--
.../hadoop/ozone/om/helpers/TestOmKeyArgs.java | 28 +-
hadoop-ozone/dev-support/checks/bats.sh | 1 +
hadoop-ozone/dist/pom.xml | 2 +-
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
.../src/main/compose/ozonesecure-mr/docker-config | 2 +-
.../src/main/compose/ozonesecure/docker-config | 8 +-
hadoop-ozone/dist/src/main/compose/restart/test.sh | 5 +
.../main/compose/upgrade/compose/ha/docker-config | 15 +
.../src/main/k8s/definitions/ozone/config.yaml | 1 +
.../main/k8s/examples/ozone/config-configmap.yaml | 1 +
.../k8s/examples/ozone/datanode-statefulset.yaml | 2 +
.../main/k8s/examples/ozone/om-statefulset.yaml | 1 +
.../main/k8s/examples/ozone/s3g-statefulset.yaml | 1 +
.../main/k8s/examples/ozone/scm-statefulset.yaml | 2 +
.../dist/src/main/k8s/examples/ozone/test.sh | 14 +-
hadoop-ozone/dist/src/main/smoketest/README.md | 4 +-
.../dist/src/main/smoketest/basic/links.robot | 7 +
.../src/main/smoketest/basic/ozone-shell-lib.robot | 20 +-
.../src/main/smoketest/basic/ozone-shell.robot | 3 +
.../freon/{validate.robot => generate-chunk.robot} | 9 +-
.../dist/src/main/smoketest/freon/generate.robot | 3 -
.../freon/{validate.robot => validate-chunk.robot} | 5 +-
.../dist/src/main/smoketest/freon/validate.robot | 3 -
.../dist/src/main/smoketest/ozone-lib/freon.robot | 4 +
.../dist/src/main/smoketest/s3/bucketcreate.robot | 10 +
.../dist/src/main/smoketest/s3/commonawslib.robot | 3 +-
.../smoketest/security/ozone-secure-tenant.robot | 2 +-
.../dist/src/main/smoketest/upgrade/finalize.robot | 2 +-
.../org/apache/hadoop/ozone/OzoneChaosCluster.java | 5 -
.../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 206 +++++++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 2 +
.../hdds/scm/TestSCMDbCheckpointServlet.java | 4 -
.../hdds/scm/storage/TestContainerCommandsEC.java | 359 +++++++++++++--
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 14 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 17 +
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 4 -
.../hadoop/ozone/TestOzoneConfigurationFields.java | 1 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 8 +-
.../hadoop/ozone/TestStorageContainerManager.java | 20 +-
.../rpc/TestContainerStateMachineFailures.java | 1 +
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +-
.../TestRefreshVolumeUsageHandler.java | 13 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 2 +-
.../server/TestSecureContainerServer.java | 2 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 254 +++++++++++
.../org/apache/hadoop/ozone/om/TestOmInit.java | 5 -
.../hadoop/ozone/om/TestSecureOzoneManager.java | 4 -
.../ozone/scm/TestStorageContainerManagerHA.java | 76 +++-
.../scm/node/TestDecommissionAndMaintenance.java | 2 +-
.../src/main/proto/OmClientProtocol.proto | 1 +
hadoop-ozone/ozone-manager/pom.xml | 6 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 6 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 465 +++++++++++--------
.../java/org/apache/hadoop/ozone/om/OMMXBean.java | 5 +
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 3 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 18 +-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 190 +++++++-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 22 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 18 +-
.../MultiTenantAccessAuthorizerRangerPlugin.java | 10 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 13 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 8 +
.../request/bucket/OMBucketSetPropertyRequest.java | 6 +
.../file/OMDirectoryCreateRequestWithFSO.java | 4 +-
.../request/file/OMFileCreateRequestWithFSO.java | 2 +-
.../ozone/om/request/file/OMFileRequest.java | 65 ++-
.../request/key/OMAllocateBlockRequestWithFSO.java | 7 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 7 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 2 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 7 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 2 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 2 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 6 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 4 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 7 +-
.../request/key/acl/OMKeyAddAclRequestWithFSO.java | 8 +-
.../key/acl/OMKeyRemoveAclRequestWithFSO.java | 5 +-
.../request/key/acl/OMKeySetAclRequestWithFSO.java | 5 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 6 +
.../S3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../S3MultipartUploadCompleteRequest.java | 26 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 9 +-
.../request/s3/security/S3SecretRequestHelper.java | 2 +-
.../om/request/upgrade/OMCancelPrepareRequest.java | 6 +-
.../request/upgrade/OMFinalizeUpgradeRequest.java | 13 +-
.../file/OMDirectoryCreateResponseWithFSO.java | 21 +-
.../response/file/OMFileCreateResponseWithFSO.java | 20 +-
.../key/OMAllocateBlockResponseWithFSO.java | 10 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 7 +-
.../response/key/OMKeyCreateResponseWithFSO.java | 10 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 15 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 14 +-
.../response/key/acl/OMKeyAclResponseWithFSO.java | 17 +-
.../S3InitiateMultipartUploadResponseWithFSO.java | 16 +-
.../S3MultipartUploadCompleteResponse.java | 3 +
.../S3MultipartUploadCompleteResponseWithFSO.java | 8 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 101 ++---
.../webapps/ozoneManager/om-overview.html | 14 +
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 2 +-
.../hadoop/ozone/om/TestOpenKeyCleanupService.java | 224 +++++++++
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 50 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 13 +-
.../ozone/om/request/OMRequestTestUtils.java | 38 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 66 +++
.../ozone/om/request/key/TestOMKeyRequest.java | 3 +-
.../om/request/key/TestOMPrefixAclRequest.java | 66 +++
.../s3/security/TestS3GetSecretRequest.java | 18 +-
.../file/TestOMDirectoryCreateResponseWithFSO.java | 2 +-
.../file/TestOMFileCreateResponseWithFSO.java | 4 +-
.../response/key/TestOMAllocateBlockResponse.java | 11 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 9 +-
.../om/response/key/TestOMKeyCommitResponse.java | 14 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 5 +-
.../om/response/key/TestOMKeyCreateResponse.java | 17 +-
.../key/TestOMKeyCreateResponseWithFSO.java | 4 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 18 +-
.../key/TestOMKeyDeleteResponseWithFSO.java | 5 +-
.../om/response/key/TestOMKeysDeleteResponse.java | 38 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 180 ++++++++
...stS3InitiateMultipartUploadResponseWithFSO.java | 12 +-
.../s3/multipart/TestS3MultipartResponse.java | 14 +-
.../TestS3MultipartUploadAbortResponse.java | 11 +-
.../TestS3MultipartUploadAbortResponseWithFSO.java | 9 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 15 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 12 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 41 +-
.../scm/ReconStorageContainerManagerFacade.java | 33 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 1 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 52 +--
.../signature/TestAuthorizationV4QueryParser.java | 10 +-
.../org/apache/hadoop/ozone/debug/ListTables.java | 7 +-
.../apache/hadoop/ozone/debug/RocksDBUtils.java | 5 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 4 +-
.../hadoop/ozone/segmentparser/RatisLogParser.java | 5 -
.../org/apache/hadoop/ozone/shell/OzoneShell.java | 9 +-
.../org/apache/hadoop/ozone/shell/s3/S3Shell.java | 9 +-
.../hadoop/ozone/shell/tenant/TenantShell.java | 9 +-
pom.xml | 16 +-
350 files changed, 11318 insertions(+), 4217 deletions(-)
copy hadoop-ozone/dist/src/main/smoketest/freon/validate.robot => .github/workflows/pull-request.yml (65%)
create mode 100644 dev-support/ci/pr_title_check.bats
create mode 100755 dev-support/ci/pr_title_check.sh
create mode 100644 hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/README.gRPC.md
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/DefaultConfigManager.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECContainerOperationClient.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
rename {hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security => hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token}/OzoneBlockTokenSecretManager.java (96%)
rename {hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security => hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token}/TestOzoneBlockTokenSecretManager.java (98%)
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ContainerReplicaCount.java => RatisContainerReplicaCount.java} (85%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/AbstractOverReplicationHandler.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/ozone/ha/TestOzoneNetUtils.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerHealthCheck.java} (50%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerHealthResult.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaCount.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerHealthCheck.java
rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ => replication}/ECContainerReplicaCount.java (63%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECOverReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java
copy hadoop-hdds/{client/src/main/java/org/apache/hadoop/hdds/scm/package-info.java => server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InflightType.java} (81%)
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/UnhealthyReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeAddressUpdateHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECContainerHealthCheck.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECContainerReplicaCount.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECOverReplicationHandler.java
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java
rename hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/{TestReplicationManager.java => replication/TestLegacyReplicationManager.java} (94%)
rename hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/{states/TestContainerReplicaCount.java => replication/TestRatisContainerReplicaCount.java} (65%)
create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
delete mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECContainerReplicaCount.java
copy hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNetUtils.java => hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyArgs.java (61%)
copy hadoop-ozone/dist/src/main/smoketest/freon/{validate.robot => generate-chunk.robot} (83%)
copy hadoop-ozone/dist/src/main/smoketest/freon/{validate.robot => validate-chunk.robot} (89%)
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSBucketLayout.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOpenKeyCleanupService.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org