You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2026/07/14 05:39:48 UTC
(ozone) 01/02: Merge remote-tracking branch 'origin/master' into HDDS-8342-object-ttl
This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a commit to branch HDDS-8342
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 3d6aa1bd0fcca842ac6d2fccbb48d9cb167d06b6
Merge: daef2f19c29 c35148d8561
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Mon Jul 13 20:16:34 2026 +0800
Merge remote-tracking branch 'origin/master' into HDDS-8342-object-ttl
.asf.yaml | 22 +-
.github/dependabot.yml | 56 +-
.../{pull-request.yml => asf-allowlist-check.yaml} | 37 +-
.github/workflows/build-ratis.yml | 26 +-
.github/workflows/check.yml | 8 +-
.github/workflows/ci-with-ratis.yml | 2 +-
.github/workflows/ci.yml | 19 +-
.github/workflows/close-stale-prs.yaml | 2 +-
.github/workflows/generate-config-doc.yml | 4 +-
.github/workflows/intermittent-test-check.yml | 50 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/populate-cache.yml | 34 +-
.github/workflows/pull-request.yml | 6 +-
.github/workflows/repeat-acceptance.yml | 14 +-
.github/workflows/update-ozone-site-config-doc.yml | 2 +-
.github/workflows/zizmor.yml | 14 +-
.mvn/extensions.xml | 4 +-
AGENTS.md | 227 +
CLAUDE.md | 1 +
NOTICE.txt | 2 +-
SECURITY.md | 10 +
THREAT_MODEL.md | 411 ++
dev-support/ci/maven-settings.xml | 7 +
dev-support/ci/selective_ci_checks.bats | 22 +
dev-support/ci/selective_ci_checks.sh | 2 +-
dev-support/ci/xml_to_md.py | 11 +
dev-support/pom.xml | 2 +-
dev-support/rat/rat-exclusions.txt | 5 +
hadoop-hdds/annotations/pom.xml | 4 +-
hadoop-hdds/cli-common/pom.xml | 4 +-
.../hadoop/hdds/cli/DeprecatedCliOption.java | 111 +
.../org/apache/hadoop/hdds/cli/GenericCli.java | 11 +-
.../hdds/cli/TestGenericCliConfiguration.java | 73 +
hadoop-hdds/client/pom.xml | 4 +-
.../hadoop/hdds/scm/ContainerClientMetrics.java | 28 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 44 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 2 +-
.../hdds/scm/storage/StreamBlockInputStream.java | 77 +-
.../hdds/scm/storage/MockDatanodePipeline.java | 288 ++
.../scm/storage/TestBlockDataStreamOutput.java | 266 ++
.../scm/storage/TestStreamBlockInputStream.java | 384 +-
hadoop-hdds/common/pom.xml | 10 +-
hadoop-hdds/common/src/main/conf/ozone-env.sh | 106 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 10 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 15 +
.../java/org/apache/hadoop/hdds/NodeDetails.java | 6 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 11 -
.../apache/hadoop/hdds/protocol/DatanodeID.java | 17 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 20 +-
.../hadoop/hdds/ratis/conf/RatisClientConfig.java | 32 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 26 +-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 3 +-
.../hadoop/hdds/scm/container/ContainerID.java | 15 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 19 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 10 +-
.../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 53 +-
.../apache/hadoop/hdds/scm/net/HostAndPort.java | 85 +
.../hadoop/hdds/scm/pipeline/PipelineID.java | 15 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 29 +
.../apache/hadoop/hdds/tracing/TracingUtil.java | 102 +-
.../hadoop/hdds/utils/ConnectionFailureUtils.java | 131 +
.../apache/hadoop/hdds/utils/SlidingWindow.java | 2 +-
.../apache/hadoop/hdds/utils/db/StringCodec.java | 11 +-
.../hadoop/hdds/utils/db/StringCodecBase.java | 47 +-
.../org/apache/hadoop/io_/retry/CallReturn.java | 69 +
.../hadoop/io_/retry/RetryInvocationHandler.java | 442 ++
.../org/apache/hadoop/io_/retry/RetryPolicies.java | 421 ++
.../org/apache/hadoop/io_/retry/RetryProxy.java | 69 +
.../ipc_/AsyncCallLimitExceededException.java | 36 -
.../org/apache/hadoop/ipc_/CallQueueManager.java | 75 -
.../main/java/org/apache/hadoop/ipc_/Client.java | 138 +-
.../apache/hadoop/ipc_/GenericRefreshProtocol.java | 49 -
.../ipc_/ObserverRetryOnActiveException.java | 31 -
.../java/org/apache/hadoop/ipc_/ProtoUtil.java | 43 -
.../org/apache/hadoop/ipc_/ProtobufRpcEngine.java | 132 +-
.../ProtocolMetaInfoServerSideTranslatorPB.java | 121 -
.../apache/hadoop/ipc_/ProtocolMetaInterface.java | 38 -
.../java/org/apache/hadoop/ipc_/ProtocolProxy.java | 82 +-
.../org/apache/hadoop/ipc_/ProtocolSignature.java | 253 --
.../java/org/apache/hadoop/ipc_/ProxyCombiner.java | 151 -
.../src/main/java/org/apache/hadoop/ipc_/RPC.java | 407 --
.../hadoop/ipc_/RefreshCallQueueProtocol.java | 44 -
.../org/apache/hadoop/ipc_/RefreshHandler.java | 32 -
.../org/apache/hadoop/ipc_/RefreshRegistry.java | 134 -
.../org/apache/hadoop/ipc_/RefreshResponse.java | 76 -
.../java/org/apache/hadoop/ipc_/RetryCache.java | 391 --
.../java/org/apache/hadoop/ipc_/RpcClientUtil.java | 241 -
.../java/org/apache/hadoop/ipc_/RpcEngine.java | 50 -
.../main/java/org/apache/hadoop/ipc_/Server.java | 454 +-
.../hadoop/ipc_/UnexpectedServerException.java | 48 -
.../org/apache/hadoop/ipc_/VersionedProtocol.java | 54 -
.../hadoop/ipc_/metrics/RetryCacheMetrics.java | 92 -
.../hadoop/ipc_/metrics/RpcDetailedMetrics.java | 6 -
.../org/apache/hadoop/ipc_/metrics/RpcMetrics.java | 33 -
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 12 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 4 +
.../hadoop/ozone/common/ChecksumByteBuffer.java | 6 +-
.../org/apache/hadoop/ozone/util/UUIDUtil.java | 13 +-
.../common/src/main/resources/ozone-default.xml | 180 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 15 +
.../apache/hadoop/hdds/ratis/TestRatisHelper.java | 15 +
.../hdds/ratis/conf/TestRatisClientConfig.java | 70 +
.../hadoop/hdds/scm/container/TestContainerID.java | 133 +
.../hdds/scm/container/TestContainerInfo.java | 4 +-
.../container/common/helpers/TestExcludeList.java | 4 +-
.../hadoop/hdds/tracing/TestLoopSampler.java | 89 +
.../hadoop/hdds/tracing/TestSpanSampling.java | 82 +
.../hadoop/hdds/tracing/TestTracingConfig.java | 100 +
.../hadoop/hdds/tracing/TestTracingUtil.java | 146 +-
.../hdds/utils/TestConnectionFailureUtils.java | 165 +
.../hadoop/hdds/utils/TestRetriableTask.java | 2 +-
.../hadoop/hdds/utils/TestSlidingWindow.java | 6 +-
.../apache/hadoop/io_/retry/TestRetryProxy.java | 304 ++
.../hadoop/io_/retry/UnreliableImplementation.java | 83 +
.../hadoop/io_/retry/UnreliableInterface.java | 66 +
.../org/apache/ratis/util/RatisUtilTestUtil.java | 16 +-
hadoop-hdds/config/pom.xml | 4 +-
hadoop-hdds/container-service/pom.xml | 4 +-
.../java/org/apache/hadoop/ozone/DNMXBean.java | 7 +
.../java/org/apache/hadoop/ozone/DNMXBeanImpl.java | 10 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 22 +-
.../checksum/DNContainerOperationClient.java | 3 +-
.../helpers/BlockDeletingServiceMetrics.java | 4 +-
.../container/common/helpers/ContainerUtils.java | 20 +-
.../common/impl/BlockDeletingService.java | 14 +-
.../ozone/container/common/impl/ContainerSet.java | 128 +-
.../container/common/impl/HddsDispatcher.java | 1 +
.../common/statemachine/DatanodeConfiguration.java | 85 +-
.../common/statemachine/DatanodeQueueMetrics.java | 18 +-
.../common/statemachine/DatanodeStateMachine.java | 19 +-
.../common/statemachine/EndpointStateMachine.java | 29 +-
.../common/statemachine/SCMConnectionManager.java | 21 +-
.../common/statemachine/StateContext.java | 48 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 28 +-
.../ReplicateContainerCommandHandler.java | 34 +-
.../common/states/datanode/InitDatanodeState.java | 15 +-
.../common/transport/server/XceiverServerGrpc.java | 11 +
.../server/ratis/ContainerStateMachine.java | 59 +-
.../container/common/utils/ContainerLogger.java | 32 +-
.../container/common/utils/DiskCheckUtil.java | 24 +-
.../container/common/utils/StorageVolumeUtil.java | 28 +
.../common/volume/AvailableSpaceFilter.java | 19 +-
.../volume/CapacityVolumeChoosingPolicy.java | 31 +-
.../ozone/container/common/volume/HddsVolume.java | 82 +-
.../container/common/volume/StorageVolume.java | 11 +-
.../container/common/volume/VolumeInfoMetrics.java | 68 +-
.../diskbalancer/DiskBalancerConfiguration.java | 2 +-
.../diskbalancer/DiskBalancerService.java | 174 +-
.../DiskBalancerVolumeCalculation.java | 56 +-
.../container/diskbalancer/DiskBalancerYaml.java | 57 +-
.../policy/DefaultContainerChoosingPolicy.java | 14 +
.../container/keyvalue/KeyValueContainer.java | 27 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 128 +-
.../keyvalue/impl/MappedBufferManager.java | 23 +-
.../statemachine/background/BlockDeletingTask.java | 52 +-
.../StaleRecoveringContainerScrubbingService.java | 10 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 36 +-
.../container/ozoneimpl/ContainerScanHelper.java | 62 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 5 +-
.../container/ozoneimpl/ScanTransientIOUtil.java | 85 +
.../replication/AbstractReplicationTask.java | 20 -
.../replication/CopyContainerResponseStream.java | 48 -
.../replication/DownloadAndImportReplicator.java | 108 -
.../replication/GrpcContainerUploader.java | 2 +-
.../replication/GrpcReplicationClient.java | 127 +-
.../replication/GrpcReplicationService.java | 57 +-
.../container/replication/ReplicationServer.java | 26 +-
.../replication/ReplicationSupervisor.java | 51 +-
.../container/replication/ReplicationTask.java | 26 +-
.../replication/SimpleContainerDownloader.java | 139 -
.../commands/ReplicateContainerCommand.java | 85 +-
.../webapps/hddsDatanode/dn-overview.html | 8 +-
.../src/main/resources/webapps/hddsDatanode/dn.js | 4 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 25 +
.../ozone/container/common/ContainerTestUtils.java | 5 +-
...stStaleRecoveringContainerScrubbingService.java | 6 +-
.../common/helpers/TestContainerUtils.java | 222 +
.../common/impl/TestContainerDataYaml.java | 38 +-
.../container/common/impl/TestContainerSet.java | 98 +
.../container/common/impl/TestHddsDispatcher.java | 151 +
.../statemachine/TestDatanodeConfiguration.java | 46 +-
.../statemachine/TestSCMConnectionManager.java | 49 +
.../common/statemachine/TestStateContext.java | 37 +-
.../TestDeleteBlocksCommandHandler.java | 89 +-
.../TestDeleteContainerCommandHandler.java | 14 +-
.../TestReconcileContainerCommandHandler.java | 2 +-
.../TestReplicateContainerCommandHandler.java | 23 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 5 +-
.../server/ratis/TestContainerStateMachine.java | 25 +-
.../container/common/utils/TestDiskCheckUtil.java | 31 +
.../common/volume/TestAvailableSpaceFilter.java | 177 +
.../volume/TestCapacityVolumeChoosingPolicy.java | 75 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 2 +-
.../common/volume/TestStorageVolumeChecker.java | 16 +-
.../volume/TestStorageVolumeHealthChecks.java | 4 +-
.../common/volume/TestVolumeInfoMetrics.java | 36 +-
.../common/volume/TestVolumeSetDiskChecks.java | 4 +-
.../diskbalancer/DiskBalancerServiceTestImpl.java | 8 +
.../TestDefaultContainerChoosingPolicy.java | 16 +
.../TestDiskBalancerProtocolServer.java | 4 +
.../diskbalancer/TestDiskBalancerService.java | 115 +-
.../diskbalancer/TestDiskBalancerTask.java | 203 +-
.../TestDiskBalancerVolumeCalculation.java | 168 +
...tDiskBalancerWithConcurrentBackgroundTasks.java | 604 +++
.../diskbalancer/TestDiskBalancerYaml.java | 53 +
.../container/keyvalue/TestKeyValueContainer.java | 95 +
.../container/keyvalue/TestKeyValueHandler.java | 104 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 14 +-
.../TestBackgroundContainerDataScanner.java | 27 +
.../TestBackgroundContainerMetadataScanner.java | 24 +
.../ozoneimpl/TestScanTransientIOUtil.java | 82 +
.../ReplicationSupervisorSchedulingBenchmark.java | 60 +-
.../TestCopyContainerResponseStream.java | 50 -
.../TestDownloadAndImportReplicator.java | 119 -
.../replication/TestGrpcReplicationService.java | 66 +-
.../replication/TestMeasuredReplicator.java | 25 +-
.../replication/TestReplicationConfig.java | 64 +-
.../replication/TestReplicationSupervisor.java | 288 +-
.../TestSendContainerRequestHandler.java | 25 +
.../replication/TestSimpleContainerDownloader.java | 239 -
hadoop-hdds/crypto-api/pom.xml | 4 +-
hadoop-hdds/crypto-default/pom.xml | 4 +-
hadoop-hdds/docs/content/design/diskbalancer.md | 6 +-
.../content/design/dn-min-space-configuration.md | 120 +
hadoop-hdds/docs/content/design/ozone-sts.md | 22 +-
.../docs/content/feature/ContainerBalancer.md | 8 +-
hadoop-hdds/docs/content/feature/DiskBalancer.md | 28 +-
.../docs/content/feature/DiskBalancer.zh.md | 30 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 2 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 4 +-
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 2 +-
.../feature/Snapshot-Configuration-Properties.md | 6 +-
hadoop-hdds/docs/content/feature/Snapshot.md | 18 +
hadoop-hdds/docs/content/interface/ReconApi.md | 1004 +++-
hadoop-hdds/docs/content/interface/S3.md | 1 -
hadoop-hdds/docs/content/tools/Repair.md | 4 +-
hadoop-hdds/docs/content/tools/TestTools.md | 14 +-
hadoop-hdds/docs/content/tools/TestTools.zh.md | 13 +-
hadoop-hdds/docs/content/tools/_index.md | 2 +-
hadoop-hdds/docs/content/tools/debug/Ldb.md | 4 +-
hadoop-hdds/docs/content/tools/debug/Ldb.zh.md | 4 +-
.../docs/content/tools/debug/RatisLogParser.md | 2 +-
hadoop-hdds/docs/pom.xml | 4 +-
.../static/swagger-resources/recon-api.yaml | 1246 ++++-
hadoop-hdds/erasurecode/pom.xml | 4 +-
.../ozone/erasurecode/rawcoder/CoderUtil.java | 16 +-
.../ozone/erasurecode/rawcoder/TestCoderUtil.java | 100 +
hadoop-hdds/framework/pom.xml | 4 +-
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 13 +-
...DiskBalancerProtocolClientSideTranslatorPB.java | 13 +-
.../ReconfigureProtocolClientSideTranslatorPB.java | 13 +-
.../SCMSecurityProtocolClientSideTranslatorPB.java | 2 +-
.../SecretKeyProtocolClientSideTranslatorPB.java | 2 +-
.../apache/hadoop/hdds/scm/ha/SequenceIdType.java | 143 +
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 3 +-
...lockLocationProtocolClientSideTranslatorPB.java | 2 +-
...inerLocationProtocolClientSideTranslatorPB.java | 8 +-
.../scm/proxy/SCMFailoverProxyProviderBase.java | 131 +-
.../apache/hadoop/hdds/scm/proxy/SCMProxyInfo.java | 18 +
.../symmetric/DefaultSecretKeySignerClient.java | 2 +-
.../certificate/authority/CertificateStore.java | 4 -
.../FixedThreadPoolWithAffinityExecutor.java | 2 +-
.../hadoop/hdds/utils/BackgroundService.java | 28 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 2 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 29 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 1 +
.../hdds/utils/db/FixedLengthStringCodec.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 32 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 35 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 11 +-
.../src/main/resources/webapps/static/ozone.js | 2 +
.../resources/webapps/static/templates/menu.html | 3 +
.../webapps/static/templates/overview.html | 6 +-
.../hdds/fs/TestCachingSpaceUsageSource.java | 15 +
.../hadoop/hdds/scm/ha/TestSequenceIdType.java | 70 +
.../proxy/TestSCMFailoverProxyProviderRefresh.java | 134 +
.../TestSCMFailoverProxyProviderRefreshWired.java | 171 +
.../hadoop/hdds/server/events/TestEventQueue.java | 46 +
.../org/apache/hadoop/hdds/utils/db/TestCodec.java | 18 +-
.../utils/db/TestRDBStoreCodecBufferIterator.java | 34 +-
.../apache/hadoop/hdds/utils/db/TestRDBTable.java | 109 +
hadoop-hdds/hadoop-dependency-client/pom.xml | 8 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
.../src/main/proto/ScmAdminProtocol.proto | 1 +
.../interface-admin/src/main/resources/proto.lock | 158 +-
hadoop-hdds/interface-client/pom.xml | 22 +-
.../src/main/proto/DatanodeClientProtocol.proto | 8 +-
.../src/main/proto/ProtocolInfo.proto | 89 -
.../interface-client/src/main/proto/hdds.proto | 4 +-
.../interface-client/src/main/resources/proto.lock | 965 +++-
hadoop-hdds/interface-server/pom.xml | 22 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 6 +-
.../interface-server/src/main/resources/proto.lock | 88 +-
hadoop-hdds/managed-rocksdb/pom.xml | 4 +-
.../db/managed/ManagedBlockBasedTableConfig.java | 8 +
.../hdds/utils/db/managed/ManagedBloomFilter.java | 14 +
.../hdds/utils/db/managed/ManagedDBOptions.java | 21 +-
.../hdds/utils/db/managed/ManagedRocksDB.java | 114 +-
.../utils/db/managed/TestManagedDBOptions.java | 59 +
hadoop-hdds/pom.xml | 4 +-
hadoop-hdds/rocks-native/pom.xml | 43 +-
.../src/main/patches/rocks-native.patch | 35 +-
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 4 +-
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 7 +
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 83 +-
hadoop-hdds/server-scm/pom.xml | 4 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 81 +-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 23 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 4 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 4 +-
.../scm/block/DeletedBlockLogStateManager.java | 4 -
.../scm/block/DeletedBlockLogStateManagerImpl.java | 24 +-
.../SCMDeletedBlockTransactionStatusManager.java | 10 +-
.../container/AbstractContainerReportHandler.java | 17 +-
.../scm/container/CloseContainerEventHandler.java | 3 +-
.../hdds/scm/container/ContainerManager.java | 33 +-
.../hdds/scm/container/ContainerManagerImpl.java | 28 +-
.../hdds/scm/container/ContainerReplica.java | 9 +
.../hdds/scm/container/ContainerReportHandler.java | 12 +-
.../hdds/scm/container/ContainerStateManager.java | 15 +-
.../scm/container/ContainerStateManagerImpl.java | 65 +-
.../IncrementalContainerReportHandler.java | 8 +-
.../scm/container/balancer/ContainerBalancer.java | 9 +-
.../ContainerBalancerSelectionCriteria.java | 4 +
.../container/balancer/ContainerBalancerTask.java | 24 +-
.../container/placement/metrics/SCMMetrics.java | 4 +-
.../replication/ContainerReplicaPendingOps.java | 22 +-
.../ContainerReplicaPendingOpsSubscriber.java | 10 +
.../replication/ECMisReplicationHandler.java | 16 +-
.../replication/ECUnderReplicationHandler.java | 22 +-
.../replication/RatisMisReplicationHandler.java | 12 +-
.../replication/RatisUnderReplicationHandler.java | 21 +-
.../container/replication/ReplicationManager.java | 35 +-
.../RatisUnhealthyReplicationCheckHandler.java | 6 +
.../scm/container/states/ContainerStateMap.java | 4 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 3 -
.../hdds/scm/ha/SCMHADBTransactionBuffer.java | 7 +
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 72 +-
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 13 +
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 164 -
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 16 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 20 +-
.../scm/ha/SCMHATransactionBufferMonitorTask.java | 26 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 18 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 25 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 80 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 131 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 13 +-
.../hdds/scm/ha/StatefulServiceDefinition.java | 37 +-
.../scm/ha/StatefulServiceStateManagerImpl.java | 5 +-
.../scm/ha/invoker/CertificateStoreInvoker.java | 161 +
.../ha/invoker/ContainerStateManagerInvoker.java | 26 +-
.../invoker/FinalizationStateManagerInvoker.java | 153 +
.../ha/invoker/RootCARotationHandlerInvoker.java | 161 +
.../hadoop/hdds/scm/ha/invoker/ScmInvoker.java | 50 +-
.../hdds/scm/ha/invoker/SecretKeyStateInvoker.java | 127 +
.../SequenceIdGeneratorStateManagerInvoker.java | 112 +
.../StatefulServiceStateManagerInvoker.java | 123 +
.../hadoop/hdds/scm/ha/io/ScmCodecFactory.java | 2 +
.../apache/hadoop/hdds/scm/ha/io/ScmListCodec.java | 6 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 5 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 5 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 51 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 16 +-
.../hdds/scm/node/PendingContainerTracker.java | 157 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 125 +-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 18 +
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 10 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 91 +-
.../hdds/scm/pipeline/ECPipelineProvider.java | 21 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 23 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 44 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 10 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 4 -
.../hdds/scm/pipeline/RatisPipelineProvider.java | 23 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 27 +-
.../scm/pipeline/WritableECContainerProvider.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 5 +-
.../safemode/AbstractContainerSafeModeRule.java | 26 +-
.../scm/safemode/ECMinDataNodeSafeModeRule.java | 151 +
.../scm/safemode/HealthyPipelineSafeModeRule.java | 51 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 19 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 61 +
.../hadoop/hdds/scm/safemode/SafeModeMetrics.java | 37 +-
.../hdds/scm/safemode/SafeModeRuleFactory.java | 50 +-
.../hdds/scm/security/RootCARotationHandler.java | 1 +
.../scm/security/RootCARotationHandlerImpl.java | 3 +-
.../hdds/scm/security/RootCARotationManager.java | 10 +-
.../scm/security/ScmSecretKeyStateBuilder.java | 3 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 3 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 102 +-
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 2 +
.../hdds/scm/server/SCMSecurityProtocolServer.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 10 +
.../upgrade/FinalizationStateManagerImpl.java | 4 +-
.../main/resources/webapps/scm/scm-overview.html | 4 +-
.../src/main/resources/webapps/scm/scm.js | 77 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 7 +-
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 38 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 2 +
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 119 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 25 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 86 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 34 +-
.../container/TestCloseContainerEventHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 16 +-
.../scm/container/TestContainerReportHandler.java | 53 +-
.../scm/container/TestContainerStateManager.java | 63 +-
.../TestIncrementalContainerReportHandler.java | 5 +-
.../scm/container/TestUnknownContainerReport.java | 2 +-
.../{TestableCluster.java => MockCluster.java} | 6 +-
.../hdds/scm/container/balancer/MockedSCM.java | 12 +-
.../TestContainerBalancerDatanodeNodeLimit.java | 37 +-
.../balancer/TestContainerBalancerStatusInfo.java | 14 +-
.../balancer/TestContainerBalancerTask.java | 67 +-
.../scm/container/balancer/TestMoveManager.java | 6 +-
.../algorithms/TestContainerPlacementFactory.java | 2 +
.../TestSCMContainerPlacementCapacity.java | 4 +
.../TestSCMContainerPlacementRackAware.java | 2 +
.../TestSCMContainerPlacementRackScatter.java | 5 +
.../TestSCMContainerPlacementRandom.java | 9 +
.../TestReconcileContainerEventHandler.java | 6 +-
.../TestContainerReplicaPendingOps.java | 65 +-
...TestQuasiClosedStuckOverReplicationHandler.java | 4 +-
...estQuasiClosedStuckUnderReplicationHandler.java | 4 +-
.../TestRatisUnderReplicationHandler.java | 4 +-
.../replication/TestReplicationManager.java | 68 +-
.../TestReplicationManagerScenarios.java | 6 +-
.../replication/TestReplicationManagerUtil.java | 14 +-
.../health/TestClosingContainerHandler.java | 4 +-
.../health/TestEmptyContainerHandler.java | 3 +-
.../TestRatisUnhealthyReplicationCheckHandler.java | 18 +
.../hdds/scm/ha/TestBackgroundSCMService.java | 6 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 9 +-
.../ha/TestSCMHATransactionBufferMonitorTask.java | 275 ++
.../hdds/scm/ha/TestSequenceIDGenerator.java | 178 +-
.../scm/ha/invoker/ScmInvokerCodeGenerator.java | 61 +-
.../ha/invoker/ScmInvokerCodeGeneratorMains.java | 100 +
.../ha/invoker/TestScmInvokerCodeGenerator.java | 89 +
.../io/TestScmCodecFactoryReplicateCoverage.java | 160 +
.../hadoop/hdds/scm/ha/io/TestScmListCodec.java | 69 +
.../hdds/scm/metadata/TestSequenceIdTypeCodec.java | 78 +
.../scm/node/DatanodeAdminMonitorTestUtil.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 15 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 4 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 13 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 2 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 2 +-
.../hdds/scm/node/TestPendingContainerTracker.java | 134 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 51 +-
.../hadoop/hdds/scm/node/TestSCMNodeMetrics.java | 5 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 12 +-
.../pipeline/TestBackgroundPipelineCreator.java | 116 +
.../TestPipelineDatanodesIntersection.java | 12 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 55 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 3 +
.../scm/pipeline/TestPipelinePlacementPolicy.java | 20 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 5 +
.../pipeline/TestWritableECContainerProvider.java | 6 +-
.../AbstractContainerSafeModeRuleTest.java | 109 +-
.../safemode/TestECMinDataNodeSafeModeRule.java | 135 +
.../TestOneReplicaPipelineSafeModeRule.java | 7 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 155 +-
.../hdds/scm/safemode/TestSafeModeRuleFactory.java | 73 +-
.../scm/security/TestRootCARotationManager.java | 3 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 11 +-
.../scm/server/TestSCMClientProtocolServer.java | 52 +
.../ozone/container/common/TestEndPoint.java | 1 +
hadoop-hdds/test-utils/pom.xml | 4 +-
.../org/apache/ozone/test/GenericTestUtils.java | 4 -
.../ozone/test/{TestClock.java => MockClock.java} | 10 +-
hadoop-ozone/cli-admin/pom.xml | 4 +-
.../hdds/scm/cli/ContainerBalancerCommands.java | 4 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 12 +-
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 2 +-
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 8 +-
.../hdds/scm/cli/datanode/DatanodeParameters.java | 20 +-
.../cli/datanode/DecommissionStatusSubCommand.java | 7 +-
.../scm/cli/datanode/DiskBalancerCommands.java | 17 +-
.../cli/datanode/DiskBalancerReportSubcommand.java | 81 +-
.../cli/datanode/DiskBalancerStartSubcommand.java | 2 +-
.../cli/datanode/DiskBalancerStatusSubcommand.java | 9 +-
.../cli/datanode/DiskBalancerStopSubcommand.java | 2 +-
.../cli/datanode/DiskBalancerSubCommandUtil.java | 2 +-
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 2 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 2 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 12 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 10 +-
.../scm/cli/pipeline/FilterPipelineOptions.java | 2 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 2 +-
.../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 2 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 8 +-
.../hadoop/ozone/admin/om/OmAddressOptions.java | 27 +-
.../hadoop/ozone/admin/om/PrepareSubCommand.java | 8 +-
.../ozone/admin/om/TransferOmLeaderSubCommand.java | 2 +-
.../ozone/admin/scm/DecommissionScmSubcommand.java | 2 +-
.../admin/scm/TransferScmLeaderSubCommand.java | 2 +-
.../hdds/cli/TestOzoneAdminDeprecatedOptions.java | 83 +
.../hdds/scm/cli/container/TestInfoSubCommand.java | 6 +-
.../cli/datanode/TestDiskBalancerSubCommands.java | 140 +-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 2 +-
.../cli/pipeline/TestClosePipelinesSubCommand.java | 4 +-
.../cli/pipeline/TestListPipelinesSubCommand.java | 6 +-
hadoop-ozone/cli-debug/pom.xml | 4 +-
.../org/apache/hadoop/ozone/debug/OzoneDebug.java | 2 +-
.../datanode/container/ContainerCommands.java | 4 +-
.../container/analyze/AnalyzeSubcommand.java | 107 +
.../analyze/ContainerDirectoryScanner.java | 275 ++
.../container/analyze/ContainerDiskOccurrence.java | 61 +
.../container/analyze/ContainerScanResult.java | 52 +
.../datanode/container/analyze}/package-info.java | 4 +-
.../apache/hadoop/ozone/debug/ldb/DBScanner.java | 53 +-
.../apache/hadoop/ozone/debug/ldb/ValueSchema.java | 4 +-
.../logs/container/ContainerLogController.java | 3 +
.../debug/logs/container/ContainerLogParser.java | 16 +-
.../container/DuplicateOpenContainersCommand.java | 9 +-
.../ozone/debug/logs/container/ListContainers.java | 33 +-
.../container/utils/ContainerDatanodeDatabase.java | 63 +-
.../container/utils/ContainerLogFileParser.java | 65 +-
.../debug/logs/container/utils/SQLDBConstants.java | 58 +-
.../ozone/debug/om/ContainerToKeyMapping.java | 2 +-
.../ozone/debug/ratis/parse/BaseLogParser.java | 2 +-
.../debug/replicas/chunk/ChunkKeyHandler.java | 4 +-
.../analyze/ContainerAnalyzeTestHelper.java | 113 +
.../container/analyze/TestAnalyzeSubcommand.java | 189 +
.../analyze/TestContainerDirectoryScanner.java | 154 +
.../{s3-secret-store => cli-interactive}/pom.xml | 44 +-
.../hadoop/ozone/shell/OzoneInteractiveShell.java | 43 +-
.../apache/hadoop/ozone/shell}/package-info.java | 6 +-
hadoop-ozone/cli-repair/pom.xml | 14 +-
.../ozone/repair/ldb/RocksDBManualCompaction.java | 17 +-
.../apache/hadoop/ozone/repair/om/CompactOMDB.java | 28 +-
.../hadoop/ozone/repair/om/FSORepairTool.java | 188 +-
.../hadoop/ozone/repair/ldb/TestLdbRepair.java | 3 +-
.../hadoop/ozone/repair/om/TestCompactOMDB.java | 120 +
hadoop-ozone/cli-shell/pom.xml | 4 +-
.../ozone/shell/OzoneInteractiveWelcome.java | 87 +
.../org/apache/hadoop/ozone/shell/OzoneShell.java | 2 +-
.../java/org/apache/hadoop/ozone/shell/REPL.java | 32 +-
.../java/org/apache/hadoop/ozone/shell/Shell.java | 15 +-
.../apache/hadoop/ozone/shell/acl/AclOption.java | 2 +-
.../hadoop/ozone/shell/keys/GetKeyHandler.java | 15 +
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 2 +-
.../org/apache/hadoop/ozone/shell/s3/S3Shell.java | 2 +-
.../hadoop/ozone/shell/s3/SetS3SecretHandler.java | 2 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 2 +-
.../tenant/TenantAssignUserAccessIdHandler.java | 5 +-
.../hadoop/ozone/shell/tenant/TenantShell.java | 2 +-
.../hadoop/ozone/shell/keys/TestGetKeyHandler.java | 218 +
hadoop-ozone/client/pom.xml | 4 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 27 +
.../client/checksum/BaseFileChecksumHelper.java | 19 +-
.../client/checksum/ECFileChecksumHelper.java | 27 +-
.../checksum/ReplicatedFileChecksumHelper.java | 7 -
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 15 -
.../hadoop/ozone/client/io/KeyCommitOutput.java | 28 +-
.../ozone/client/io/KeyDataStreamOutput.java | 32 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 32 +-
.../ozone/client/io/KeyOutputStreamSemaphore.java | 8 +-
.../ozone/client/io/OzoneDataStreamOutput.java | 67 +-
.../hadoop/ozone/client/io/OzoneOutputStream.java | 26 +-
.../ozone/client/protocol/ClientProtocol.java | 43 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 115 +-
.../hadoop/ozone/client/TestOzoneClient.java | 34 -
.../hadoop/ozone/client/TestOzoneECClient.java | 11 +-
.../client/checksum/FileChecksumBenchmark.java | 594 +++
.../client/checksum/TestFileChecksumHelper.java | 6 +-
.../ozone/client/io/TestKeyDataStreamOutput.java | 313 ++
.../ozone/client/io/TestOzoneOutputStream.java | 45 +-
.../hadoop/ozone/client/rpc/TestRpcClient.java | 49 +
hadoop-ozone/common/pom.xml | 4 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 9 +
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 22 +-
.../apache/hadoop/ozone/om/IOmMetadataReader.java | 8 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 8 +-
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 4 +
.../om/ha/HadoopRpcOMFailoverProxyProvider.java | 37 +-
...doopRpcOMFollowerReadFailoverProxyProvider.java | 12 +-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 89 +-
.../org/apache/hadoop/ozone/om/ha/OMProxyInfo.java | 153 +-
.../hadoop/ozone/om/helpers/AclListBuilder.java | 7 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 38 +
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 40 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 4 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 15 +-
.../ozone/om/helpers/OmMultipartPartKey.java | 39 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 55 +-
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 75 +-
.../hadoop/ozone/om/protocol/OMAdminProtocol.java | 10 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 26 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 176 +-
.../protocolPB/OMAdminProtocolClientSideImpl.java | 7 +-
.../OMInterServiceProtocolClientSideImpl.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 76 +-
.../om/protocolPB/OzoneManagerProtocolPB.java | 2 +-
.../ozone/security/acl/IAccessAuthorizer.java | 16 +-
...doopRpcOMFollowerReadFailoverProxyProvider.java | 4 +-
.../TestOMFailoverProxyProviderRefreshWired.java | 225 +
.../ozone/om/ha/TestOMProxyInfoDnsRefresh.java | 168 +
.../ozone/om/helpers/TestOmMultipartPartKey.java | 29 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 102 +
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 219 +
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/datanode/pom.xml | 4 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
hadoop-ozone/dist/pom.xml | 27 +-
.../dist/src/main/compose/common/ec-test.sh | 4 +-
.../Ozone - Container Balancer Metrics.json | 1379 ++++++
...ne - Datanode Decommission and Maintenance.json | 1243 +++++
.../grafana/dashboards/Ozone - OM Overview.json | 2796 ++++++++++++
.../grafana/dashboards/Ozone - OM Snapshot.json | 2639 ++++++++++-
.../grafana/dashboards/Ozone - SCM Safemode.json | 203 +
.../grafana/dashboards/Ozone - SCM overview.json | 1766 ++++++++
.../grafana/provisioning/dashboards/dashboards.yml | 17 +-
.../provisioning/datasources/datasources.yml | 17 +-
.../dist/src/main/compose/common/hadoop-test.sh | 6 +-
.../dist/src/main/compose/common/replicas-test.sh | 141 +-
.../src/main/compose/ozone-ha/docker-compose.yaml | 25 +-
.../src/main/compose/ozone-ha/test-haproxy-s3g.sh | 6 +-
.../dist/src/main/compose/ozone-ha/test.sh | 5 +-
.../dist/src/main/compose/ozone/docker-config | 32 +-
.../dist/src/main/compose/ozone/monitoring.conf | 2 +-
.../dist/src/main/compose/ozone/monitoring.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 7 +-
.../main/compose/ozonescripts/docker-compose.yaml | 66 +-
.../compose/ozonesecure-ha/docker-compose.yaml | 3 +
.../src/main/compose/ozonesecure-ha/docker-config | 1 +
.../main/compose/ozonesecure-ha/s3-haproxy.yaml | 3 +
.../compose/ozonesecure-ha/test-haproxy-s3g.sh | 7 +-
.../src/main/compose/ozonesecure-ha/test-ranger.sh | 2 +-
.../compose/ozonesecure-ha/test-repair-tools.sh | 30 +-
.../ozonesecure-ha/test-s3g-virtual-host.sh | 2 +-
.../dist/src/main/compose/ozonesecure-ha/test.sh | 6 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 23 -
.../dist/src/main/compose/ozonesecure-mr/README.md | 73 -
.../compose/ozonesecure-mr/docker-compose.yaml | 100 -
.../src/main/compose/ozonesecure-mr/docker-config | 88 -
.../dist/src/main/compose/ozonesecure-mr/krb5.conf | 39 -
.../main/compose/ozonesecure/docker-compose.yaml | 72 +-
.../src/main/compose/ozonesecure/docker-config | 13 +-
.../test.sh => ozonesecure/test-hadoop.sh} | 0
.../src/main/compose/ozonesecure/test-vault.sh | 3 +-
.../dist/src/main/compose/ozonesecure/test.sh | 6 +-
.../dist/src/main/compose/ozonesecure/vault.yaml | 1 -
.../src/main/compose/restart/docker-compose.yaml | 15 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 10 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 4 +-
.../dist/src/main/compose/xcompat/new-cluster.yaml | 3 +-
.../definitions/ozone/definitions/persistence.yaml | 3 +
.../examples/ozone-ha/datanode-statefulset.yaml | 3 +
.../k8s/examples/ozone-ha/httpfs-statefulset.yaml | 3 +
.../main/k8s/examples/ozone-ha/om-statefulset.yaml | 3 +
.../k8s/examples/ozone-ha/recon-statefulset.yaml | 3 +
.../k8s/examples/ozone-ha/s3g-statefulset.yaml | 3 +
.../k8s/examples/ozone-ha/scm-statefulset.yaml | 3 +
.../k8s/examples/ozone/datanode-statefulset.yaml | 3 +
.../k8s/examples/ozone/httpfs-statefulset.yaml | 3 +
.../main/k8s/examples/ozone/om-statefulset.yaml | 3 +
.../main/k8s/examples/ozone/recon-statefulset.yaml | 3 +
.../main/k8s/examples/ozone/s3g-statefulset.yaml | 3 +
.../main/k8s/examples/ozone/scm-statefulset.yaml | 3 +
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 31 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 29 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 76 +-
.../src/main/smoketest/admincli/pipeline.robot | 24 +-
.../main/smoketest/{s3 => }/awss3virtualhost.robot | 5 +-
.../src/main/smoketest/balancer/testBalancer.robot | 2 +-
.../smoketest/debug/ozone-debug-keywords.robot | 43 +-
.../smoketest/debug/ozone-debug-tests-ec3-2.robot | 38 +-
.../smoketest/debug/ozone-debug-tests-ec6-3.robot | 63 +
.../smoketest/diskbalancer/testdiskbalancer.robot | 6 +-
.../src/main/smoketest/ec/awss3ecstorage.robot | 11 +-
.../dist/src/main/smoketest/ec/lib.resource | 9 +
.../src/main/smoketest/prometheus/prometheus.robot | 35 +-
.../main/smoketest/prometheus/prometheus_check.py | 76 +
.../dist/src/main/smoketest/recon/recon-api.robot | 61 +-
.../src/main/smoketest/repair/om-compact.robot | 54 -
.../src/main/smoketest/s3/MultipartUpload.robot | 37 +-
.../dist/src/main/smoketest/s3/buckettagging.robot | 89 +
.../dist/src/main/smoketest/s3/commonawslib.robot | 21 +-
.../src/main/smoketest/s3/presigned_url_helper.py | 48 +
.../src/main/smoketest/s3/s3_compatbility_check.sh | 1 +
.../main/smoketest/snapshot/snapshot-defrag.robot | 172 +
hadoop-ozone/dist/src/shell/conf/log4j.properties | 6 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 73 +-
.../dist/src/shell/ozone/ozone-functions.sh | 28 +-
.../src/test/shell/http_request_logger.bats} | 31 +-
.../dev-support/findbugsExcludeFile.xml | 24 -
.../fault-injection-test/mini-chaos-tests/pom.xml | 156 -
.../mini-chaos-tests/src/test/bin/start-chaos.sh | 57 -
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 401 --
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 148 -
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 55 -
.../ozone/TestDatanodeMiniChaosOzoneCluster.java | 57 -
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 222 -
.../TestOzoneManagerMiniChaosOzoneCluster.java | 62 -
...orageContainerManagerMiniChaosOzoneCluster.java | 62 -
.../hadoop/ozone/failure/FailureManager.java | 99 -
.../org/apache/hadoop/ozone/failure/Failures.java | 226 -
.../ozone/loadgenerators/AgedLoadGenerator.java | 77 -
.../hadoop/ozone/loadgenerators/DataBuffer.java | 52 -
.../loadgenerators/FilesystemLoadGenerator.java | 55 -
.../hadoop/ozone/loadgenerators/LoadBucket.java | 320 --
.../hadoop/ozone/loadgenerators/LoadExecutors.java | 109 -
.../hadoop/ozone/loadgenerators/LoadGenerator.java | 67 -
.../loadgenerators/NestedDirLoadGenerator.java | 55 -
.../ozone/loadgenerators/RandomLoadGenerator.java | 53 -
.../loadgenerators/ReadOnlyLoadGenerator.java | 52 -
.../services/org.apache.hadoop.fs.FileSystem | 17 -
.../src/test/resources/log4j.properties | 40 -
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 5 +-
hadoop-ozone/freon/pom.xml | 4 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../freon/FreonS3TraceContextRequestHandler.java} | 38 +-
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 12 +-
.../hadoop/ozone/freon/HadoopFsGenerator.java | 20 +-
.../hadoop/ozone/freon/HadoopFsValidator.java | 18 +-
.../ozone/freon/HadoopNestedDirGenerator.java | 5 +-
.../ozone/freon/OzoneClientKeyListReader.java | 151 +
.../ozone/freon/OzoneClientKeyValidator.java | 13 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 64 +-
.../hadoop/ozone/freon/S3EntityGenerator.java | 1 +
.../ozone/freon/TestHadoopFsClientClose.java | 138 +
.../ozone/freon/TestOzoneClientKeyListReader.java | 41 +-
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 90 +
hadoop-ozone/httpfsgateway/pom.xml | 20 +-
.../apache/ozone/fs/http/server/FSOperations.java | 106 +-
.../apache/ozone/fs/http/server/HttpFSServer.java | 13 +-
.../instrumentation/InstrumentationService.java | 64 +-
.../org/apache/ozone/lib/wsrs/JSONMapProvider.java | 9 +-
.../org/apache/ozone/lib/wsrs/JSONProvider.java | 79 -
.../TestInstrumentationSerialization.java | 78 +
.../apache/ozone/lib/wsrs/TestJSONMapProvider.java | 71 +
hadoop-ozone/iceberg/pom.xml | 128 +-
.../hadoop/ozone/iceberg/IcebergCommand.java} | 27 +-
.../ozone/iceberg/RewriteTablePathCommand.java | 127 +
.../ozone/iceberg/RewriteTablePathOzoneAction.java | 528 ++-
.../iceberg/TestRewriteTablePathOzoneAction.java | 661 ++-
hadoop-ozone/insight/pom.xml | 4 +-
.../insight/datanode/PipelineComponentUtil.java | 2 +-
hadoop-ozone/integration-test-recon/pom.xml | 4 +-
.../AbstractTestStorageDistributionEndpoint.java | 19 +-
...Utils.java => ReconOmMetaManagerTestUtils.java} | 11 +-
.../ozone/recon/TestNSSummaryMemoryLeak.java | 87 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 32 +-
.../ozone/recon/TestReconContainerEndpoint.java | 9 +-
.../TestReconContainerHealthSummaryEndToEnd.java | 1285 ++++++
.../TestReconQuasiClosedContainerEndpoint.java | 242 +
.../apache/hadoop/ozone/recon/TestReconTasks.java | 12 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 44 +-
.../recon/TestStorageDistributionEndpointEC.java | 8 +-
.../TestStorageDistributionEndpointRatis.java | 10 +-
hadoop-ozone/integration-test-s3/pom.xml | 11 +-
.../hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java | 1 +
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 9 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 219 +
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 469 ++
hadoop-ozone/integration-test/pom.xml | 4 +-
.../fs/ozone/AbstractOzoneFileSystemTest.java | 87 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 80 +-
.../hadoop/fs/ozone/OzoneFileSystemTestBase.java | 79 +
.../apache/hadoop/hdds/scm/TestCommitInRatis.java | 1 -
.../scm/TestDatanodeSCMNodesReconfiguration.java | 1 -
.../hdds/scm/TestSCMDatanodeProtocolServer.java | 3 +-
.../org/apache/hadoop/hdds/scm/TestSCMMXBean.java | 3 +
.../hdds/scm/TestStorageContainerManagerHA.java | 1 -
.../apache/hadoop/hdds/scm/TestWatchForCommit.java | 1 -
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 172 +
.../TestContainerStateManagerIntegration.java | 7 +-
.../TestPendingContainerTrackerIntegration.java | 184 +
.../scm/node/TestDecommissionAndMaintenance.java | 6 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 5 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 4 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 68 +-
.../hadoop/hdds/scm/storage/TestCommitWatcher.java | 1 -
.../hdds/scm/storage/TestContainerCommandsEC.java | 5 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 36 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 96 +
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 1 -
...stClientRetryContainerStateMachineFailures.java | 413 ++
.../ozone/client/rpc/TestClientRetryTimeout.java | 497 ++
.../rpc/TestContainerReplicationEndToEnd.java | 3 +-
.../client/rpc/TestContainerStateMachine.java | 78 +-
.../TestContainerStateMachineFailureOnRead.java | 33 +-
.../rpc/TestContainerStateMachineFailures.java | 465 +-
.../rpc/TestContainerStateMachineFlushDelay.java | 61 +-
.../client/rpc/TestDeleteWithInAdequateDN.java | 1 -
.../client/rpc/TestFailureHandlingByClient.java | 8 +-
.../rpc/TestFailureHandlingByClientFlushDelay.java | 1 -
.../client/rpc/TestHybridPipelineOnDatanode.java | 1 -
.../rpc/TestMultiBlockWritesWithDnFailures.java | 1 -
...estOzoneClientRetriesOnExceptionFlushDelay.java | 1 -
.../rpc/TestOzoneClientRetriesOnExceptions.java | 1 -
.../client/rpc/TestValidateBCSIDOnRestart.java | 105 +-
.../rpc/read/TestStreamReadDatanodeFailover.java | 287 ++
.../container/TestContainerReportHandling.java | 201 +-
.../TestContainerReportHandlingWithHA.java | 183 -
.../apache/hadoop/ozone/container/TestHelper.java | 47 +
.../commandhandler/TestBlockDeletion.java | 94 +-
.../TestCloseContainerByPipeline.java | 1 -
.../commandhandler/TestCloseContainerHandler.java | 9 +-
.../commandhandler/TestDeleteContainerHandler.java | 120 +-
.../commandhandler/TestFinalizeBlock.java | 12 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 80 -
.../ozoneimpl/TestSecureOzoneContainer.java | 2 +-
.../replication/TestContainerReplication.java | 40 -
.../container/server/TestContainerServer.java | 4 +-
.../server/TestSecureContainerServer.java | 2 +-
.../dn/TestDatanodeMinFreeSpaceIntegration.java | 121 +
...estDuplicateContainerDirScannerIntegration.java | 241 +
.../ozone/dn/ratis/TestDnRatisLogParser.java | 4 +
.../TestOnDemandContainerScannerIntegration.java | 11 +-
...agerHA.java => AbstractOzoneManagerHATest.java} | 57 +-
.../org/apache/hadoop/ozone/om/OmTestUtil.java | 8 +
.../TestOMDbCheckpointServletInodeBasedXfer.java | 76 +-
...shots.java => TestOMRatisSnapshotTransfer.java} | 587 +--
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 706 +--
.../ozone/om/TestOmContainerLocationCache.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 403 +-
.../ozone/om/TestOzoneManagerHAFollowerRead.java | 412 +-
...stOzoneManagerHAFollowerReadWithAllRunning.java | 71 +-
...OzoneManagerHAFollowerReadWithStoppedNodes.java | 16 +-
.../om/TestOzoneManagerHAWithStoppedNodes.java | 26 +-
.../om/TestOzoneManagerListVolumesSecure.java | 491 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 27 +-
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 20 +-
...mSnapshotFsoWithNativeLibWithLinkedBuckets.java | 33 -
.../om/snapshot/TestOmSnapshotObjectStore.java | 2 +-
.../om/snapshot/TestOzoneManagerHASnapshot.java | 20 +-
.../hadoop/ozone/repair/om/TestFSORepairTool.java | 25 +
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 4 +-
...skBalancerDuringDecommissionAndMaintenance.java | 23 +-
.../hadoop/ozone/shell/TestOzoneDebugShell.java | 64 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 6 +-
hadoop-ozone/interface-client/pom.xml | 4 +-
.../src/main/proto/OMAdminProtocol.proto | 4 +
.../src/main/proto/OmClientProtocol.proto | 91 +-
.../interface-client/src/main/resources/proto.lock | 344 +-
hadoop-ozone/interface-storage/pom.xml | 4 +-
.../ozone/om/helpers/TestOmKeyInfoCodec.java | 20 +-
hadoop-ozone/mini-cluster/pom.xml | 4 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 3 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 16 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 10 +-
.../hadoop/ozone/UniformDatanodesFactory.java | 8 +-
hadoop-ozone/multitenancy-ranger/pom.xml | 4 +-
hadoop-ozone/ozone-manager/pom.xml | 4 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 4 +
.../hadoop/ozone/om/DeletingServiceMetrics.java | 37 +
.../java/org/apache/hadoop/ozone/om/OMMXBean.java | 2 +
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 41 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 21 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 31 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 40 +
.../hadoop/ozone/om/OmMetadataReaderMetrics.java | 4 +
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 10 +
.../hadoop/ozone/om/OmSnapshotLocalDataYaml.java | 12 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 5 +-
.../apache/hadoop/ozone/om/OmSnapshotMetrics.java | 12 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 32 +-
.../org/apache/hadoop/ozone/om/ResolvedBucket.java | 10 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../org/apache/hadoop/ozone/om/ha/OMService.java | 51 +-
.../hadoop/ozone/om/ha/OMServiceException.java} | 24 +-
.../hadoop/ozone/om/ha/OMServiceManager.java | 77 +
.../hadoop/ozone/om/helpers/OMAuditLogger.java | 3 +
.../ozone/om/ratis/OzoneManagerRatisServer.java | 63 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 4 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 6 +
.../hadoop/ozone/om/request/OMClientRequest.java | 39 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 16 +-
.../om/request/key/OMAllocateBlockRequest.java | 82 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 184 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 25 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 4 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 3 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 62 +-
.../s3/tagging/S3BucketTaggingRequestBase.java | 183 +
.../s3/tagging/S3DeleteBucketTaggingRequest.java | 132 +
.../s3/tagging/S3PutBucketTaggingRequest.java | 129 +
.../request/snapshot/OMSnapshotCreateRequest.java | 4 +-
.../request/snapshot/OMSnapshotDeleteRequest.java | 8 +-
.../request/snapshot/OMSnapshotPurgeRequest.java | 11 +-
.../validation/OMClientVersionValidator.java | 2 +-
.../validation/OMLayoutVersionValidator.java | 2 +-
.../om/request/volume/OMQuotaRepairRequest.java | 6 +
.../om/request/volume/OMVolumeDeleteRequest.java | 40 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 35 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 39 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 45 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 8 +-
.../volume/acl/OMVolumeRemoveAclRequest.java | 8 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 8 +-
.../hadoop/ozone/om/service/CompactDBUtil.java | 34 +-
.../hadoop/ozone/om/service/CompactionService.java | 7 +-
.../ozone/om/service/DirectoryDeletingService.java | 55 +-
.../ozone/om/service/KeyLifecycleService.java | 2 +-
.../hadoop/ozone/om/service/QuotaRepairTask.java | 267 +-
.../ozone/om/service/SnapshotDeletingService.java | 14 +-
.../om/service/SnapshotDiffCleanupService.java | 33 +-
.../om/snapshot/OmSnapshotLocalDataManager.java | 83 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 4 +-
.../om/snapshot/defrag/SnapshotDefragService.java | 16 +-
.../protocolPB/OMAdminProtocolServerSideImpl.java | 6 +-
.../protocolPB/OzoneManagerRequestHandler.java | 23 +
.../main/resources/webapps/ozoneManager/index.html | 3 +-
.../webapps/ozoneManager/om-deletion.html | 186 +
.../webapps/ozoneManager/om-overview.html | 59 -
.../resources/webapps/ozoneManager/ozoneManager.js | 96 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 450 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 25 +
.../ozone/om/TestOmSnapshotLocalDataYaml.java | 36 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../hadoop/ozone/om/ha/TestOMServiceManager.java | 103 +
.../hadoop/ozone/om/lock/TestKeyPathLock.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 40 +
.../om/ratis/TestOzoneManagerStateMachine.java | 5 +
.../ozone/om/request/OMRequestTestUtils.java | 7 +
.../TestOMClientRequestUserInfoFallback.java | 106 +
...tBucketRequest.java => BucketRequestTests.java} | 2 +-
.../request/bucket/TestOMBucketCreateRequest.java | 2 +-
.../request/bucket/TestOMBucketDeleteRequest.java | 2 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 2 +-
.../bucket/acl/TestOMBucketAddAclRequest.java | 4 +-
.../bucket/acl/TestOMBucketRemoveAclRequest.java | 4 +-
.../bucket/acl/TestOMBucketSetAclRequest.java | 4 +-
.../om/request/file/TestOMFileCreateRequest.java | 4 +-
.../om/request/file/TestOMRecoverLeaseRequest.java | 4 +-
...estOMKeyRequest.java => OMKeyRequestTests.java} | 2 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 2 +-
.../TestOMDirectoriesPurgeRequestAndResponse.java | 2 +-
.../ozone/om/request/key/TestOMKeyAclRequest.java | 2 +-
.../om/request/key/TestOMKeyCommitRequest.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 2 +-
.../om/request/key/TestOMKeyDeleteRequest.java | 133 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 69 +
.../key/TestOMKeyPurgeRequestAndResponse.java | 2 +-
.../om/request/key/TestOMKeyRenameRequest.java | 2 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 2 +-
.../om/request/key/TestOMKeysRenameRequest.java | 2 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 2 +-
.../om/request/key/TestOMPrefixAclRequest.java | 2 +-
.../om/request/key/TestOMSetTimesRequest.java | 2 +-
...rtRequest.java => S3MultipartRequestTests.java} | 2 +-
.../TestS3ExpiredMultipartUploadsAbortRequest.java | 2 +-
.../TestS3InitiateMultipartUploadRequest.java | 2 +-
.../TestS3MultipartUploadAbortRequest.java | 2 +-
.../TestS3MultipartUploadCommitPartRequest.java | 2 +-
.../TestS3MultipartUploadCompleteRequest.java | 2 +-
.../tagging/TestS3DeleteBucketTaggingRequest.java | 257 ++
.../tagging/TestS3DeleteObjectTaggingRequest.java | 4 +-
.../s3/tagging/TestS3PutBucketTaggingRequest.java | 274 ++
.../s3/tagging/TestS3PutObjectTaggingRequest.java | 4 +-
...est.java => OMDelegationTokenRequestTests.java} | 2 +-
.../security/TestOMGetDelegationTokenRequest.java | 2 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 11 +-
.../snapshot/TestOMSnapshotDeleteRequest.java | 9 +-
.../TestOMSnapshotMoveTableKeysRequest.java | 4 +-
.../TestOMSnapshotPurgeRequestAndResponse.java | 11 +-
.../snapshot/TestOMSnapshotRenameRequest.java | 4 +-
...estOMSnapshotSetPropertyRequestAndResponse.java | 4 +-
.../upgrade/TestOMCancelPrepareRequest.java | 4 +-
...olumeRequest.java => OMVolumeRequestTests.java} | 2 +-
.../request/volume/TestOMVolumeCreateRequest.java | 2 +-
.../request/volume/TestOMVolumeDeleteRequest.java | 2 +-
.../volume/TestOMVolumeSetOwnerRequest.java | 2 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 2 +-
.../volume/acl/TestOMVolumeAddAclRequest.java | 4 +-
.../volume/acl/TestOMVolumeRemoveAclRequest.java | 4 +-
.../volume/acl/TestOMVolumeSetAclRequest.java | 4 +-
...ResponseUtils.java => OMResponseTestUtils.java} | 4 +-
.../bucket/TestOMBucketCreateResponse.java | 4 +-
.../bucket/TestOMBucketDeleteResponse.java | 4 +-
.../bucket/TestOMBucketSetPropertyResponse.java | 4 +-
.../file/TestOMDirectoryCreateResponse.java | 4 +-
.../file/TestOMDirectoryCreateResponseWithFSO.java | 4 +-
...tOMKeyResponse.java => OMKeyResponseTests.java} | 2 +-
.../response/key/TestOMAllocateBlockResponse.java | 2 +-
.../om/response/key/TestOMKeyCommitResponse.java | 2 +-
.../om/response/key/TestOMKeyCreateResponse.java | 2 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 2 +-
.../om/response/key/TestOMKeyRenameResponse.java | 2 +-
.../key/TestOMKeyRenameResponseWithFSO.java | 4 +-
.../om/response/key/TestOMKeysDeleteResponse.java | 2 +-
.../om/response/key/TestOMKeysRenameResponse.java | 2 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 2 +-
.../key/acl/prefix/TestOMPrefixAclResponse.java | 4 +-
...Response.java => S3MultipartResponseTests.java} | 105 +-
...TestS3ExpiredMultipartUploadsAbortResponse.java | 2 +-
.../TestS3InitiateMultipartUploadResponse.java | 2 +-
.../TestS3MultipartUploadAbortResponse.java | 2 +-
...> TestS3MultipartUploadCommitPartResponse.java} | 169 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 246 +-
.../TestS3MultipartUploadCompleteResponse.java | 388 ++
...stS3MultipartUploadCompleteResponseWithFSO.java | 450 +-
.../tagging/TestS3DeleteObjectTaggingResponse.java | 4 +-
.../s3/tagging/TestS3PutObjectTaggingResponse.java | 4 +-
...se.java => OMDelegationTokenResponseTests.java} | 2 +-
.../security/TestOMGetDelegationTokenResponse.java | 2 +-
.../TestOMSnapshotMoveTableKeysResponse.java | 4 +-
...umeResponse.java => OMVolumeResponseTests.java} | 2 +-
.../volume/TestOMVolumeCreateResponse.java | 2 +-
.../volume/TestOMVolumeDeleteResponse.java | 2 +-
.../volume/TestOMVolumeSetOwnerResponse.java | 2 +-
.../volume/TestOMVolumeSetQuotaResponse.java | 2 +-
.../om/service/TestDirectoryDeletingService.java | 29 +
.../ozone/om/service/TestQuotaRepairTask.java | 146 +-
.../om/service/TestSnapshotDeletingService.java | 119 +-
.../om/service/TestSnapshotDiffCleanupService.java | 57 +-
...e.java => SnapshotRequestAndResponseTests.java} | 6 +-
.../snapshot/TestOmSnapshotLocalDataManager.java | 49 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 173 +-
.../snapshot/defrag/TestSnapshotDefragService.java | 44 +
hadoop-ozone/ozonefs-common/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
hadoop-ozone/pom.xml | 5 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
hadoop-ozone/recon/pom.xml | 28 +-
.../apache/hadoop/ozone/recon/ReconConstants.java | 7 +
.../hadoop/ozone/recon/ReconControllerModule.java | 8 +
.../hadoop/ozone/recon/ReconRestServletModule.java | 66 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 8 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 78 +-
.../apache/hadoop/ozone/recon/TarExtractor.java | 8 +-
.../ozone/recon/api/AccessHeatMapEndpoint.java | 1 -
.../apache/hadoop/ozone/recon/api/AdminOnly.java | 34 -
.../hadoop/ozone/recon/api/BlocksEndPoint.java | 1 -
.../hadoop/ozone/recon/api/BucketEndpoint.java | 1 -
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 63 +-
.../ozone/recon/api/DataNodeMetricsService.java | 91 +-
.../hadoop/ozone/recon/api/FeaturesEndpoint.java | 1 -
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 1 -
.../ozone/recon/api/OMDBInsightEndpoint.java | 1 -
.../ozone/recon/api/PendingDeletionEndpoint.java | 13 +-
.../recon/api/StorageDistributionEndpoint.java | 69 +-
.../ozone/recon/api/TriggerDBSyncEndpoint.java | 35 +-
.../hadoop/ozone/recon/api/VolumeEndpoint.java | 1 -
.../api/types/DataNodeMetricsCompleteResponse.java | 79 +
.../api/types/DataNodeMetricsProgressResponse.java | 53 +
.../api/types/DataNodeMetricsServiceResponse.java | 130 -
.../api/types/QuasiClosedContainerMetadata.java | 124 +
.../api/types/QuasiClosedContainersResponse.java | 82 +
.../ozone/recon/chatbot/ChatbotConfigKeys.java | 173 +
.../ozone/recon/chatbot/ChatbotException.java | 42 +
.../hadoop/ozone/recon/chatbot/ChatbotModule.java | 55 +
.../ozone/recon/chatbot/agent/ChatbotAgent.java | 676 +++
.../ozone/recon/chatbot/agent/ChatbotUtils.java | 149 +
.../recon/chatbot/agent/LlmToolSpecFactory.java | 291 ++
.../ozone/recon/chatbot/agent}/package-info.java | 4 +-
.../ozone/recon/chatbot/api/ChatbotEndpoint.java | 368 ++
.../ozone/recon/chatbot/api}/package-info.java | 4 +-
.../hadoop/ozone/recon/chatbot/llm/GenParams.java} | 35 +-
.../hadoop/ozone/recon/chatbot/llm/LLMClient.java | 219 +
.../recon/chatbot/llm/LangChain4jDispatcher.java | 512 +++
.../hadoop/ozone/recon/chatbot/llm/LlmRouting.java | 130 +
.../ozone/recon/chatbot/llm}/package-info.java | 4 +-
.../hadoop/ozone/recon/chatbot}/package-info.java | 4 +-
.../recon/chatbot/recon/ReconApiAllowlist.java | 78 +
.../recon/chatbot/recon/ReconEndpointRouter.java | 249 +
.../recon/chatbot/recon/ReconQueryExecutor.java | 96 +
.../recon/chatbot/recon/ReconQueryResult.java | 65 +
.../chatbot/recon/ReconResponseUnwrapper.java | 57 +
.../ozone/recon/chatbot/recon}/package-info.java | 8 +-
.../recon/chatbot/security/CredentialHelper.java | 96 +
.../recon/chatbot/security}/package-info.java | 4 +-
.../ozone/recon/fsck/ReconReplicationManager.java | 4 +-
.../metrics/ReconScmContainerSyncMetrics.java | 217 +
.../persistence/ContainerHealthSchemaManager.java | 56 +-
.../recon/persistence/DerbyDataSourceProvider.java | 3 +-
.../ozone/recon/scm/ContainerReplicaHistory.java | 18 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 9 +-
.../ozone/recon/scm/ReconContainerManager.java | 200 +-
.../scm/ReconStorageContainerManagerFacade.java | 516 ++-
.../recon/scm/ReconStorageContainerSyncHelper.java | 738 ++-
.../recon/spi/ReconContainerMetadataManager.java | 8 +-
.../recon/spi/StorageContainerServiceProvider.java | 29 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 2 +-
.../impl/ReconContainerMetadataManagerImpl.java | 22 +-
.../impl/StorageContainerServiceProviderImpl.java | 16 +
.../recon/tasks/DataNodeMetricsCollectionTask.java | 8 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 80 +-
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 42 +
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 20 +-
.../ozone/recon/tasks/NSSummaryTaskWithOBS.java | 31 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 32 +-
.../chatbot/recon-fallback-prompt-template.txt | 10 +
.../chatbot/recon-summarization-prompt.txt | 29 +
.../recon-tool-selection-prompt-preamble.txt | 49 +
.../main/resources/chatbot/recon-tool-semantics.md | 838 ++++
.../webapps/recon/ozone-recon-web/.eslintignore | 26 -
.../webapps/recon/ozone-recon-web/.eslintrc.json | 180 +-
.../webapps/recon/ozone-recon-web/.gitignore | 6 +
.../ozone-recon-web/e2e/chatbot-errors.spec.ts | 268 ++
.../recon/ozone-recon-web/eslint.config.mjs | 109 +
.../webapps/recon/ozone-recon-web/package.json | 33 +-
.../recon/ozone-recon-web/playwright.config.ts} | 44 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 4771 +++++++++++---------
.../src/__tests__/assistant/Assistant.test.tsx | 322 ++
.../src/__tests__/capacity/Capacity.test.tsx | 217 +-
.../mocks/assistantMocks/assistantServer.ts | 158 +
.../ozone-recon-web/src/__tests__/vitest.setup.ts | 4 +-
.../webapps/recon/ozone-recon-web/src/app.tsx | 165 +-
.../src/utils/axiosRequestHelper.tsx | 14 +
.../recon/ozone-recon-web/src/utils/clipboard.ts | 47 +
.../src/v2/components/navBar/navBar.tsx | 23 +-
.../src/v2/components/plots/nuPieChart.tsx | 87 +-
.../src/v2/components/tables/containersTable.tsx | 11 +-
.../tables/insights/containerMismatchTable.tsx | 6 -
.../tables/insights/deletePendingDirsTable.tsx | 6 -
.../tables/insights/deletePendingKeysTable.tsx | 6 -
.../tables/insights/deletedContainerKeysTable.tsx | 6 -
.../src/v2/constants/breadcrumbs.constants.tsx | 4 +-
.../src/v2/constants/chatbot.constants.tsx | 130 +
.../ozone-recon-web/src/v2/hooks/useChat.hook.tsx | 201 +
.../src/v2/pages/assistant/assistant.less | 744 +++
.../src/v2/pages/assistant/assistant.tsx | 185 +
.../src/v2/pages/assistant/components/Composer.tsx | 117 +
.../pages/assistant/components/DisabledState.tsx | 50 +
.../v2/pages/assistant/components/EmptyState.tsx | 83 +
.../assistant/components/LoadingIndicator.tsx | 70 +
.../pages/assistant/components/MessageBubble.tsx | 97 +
.../v2/pages/assistant/components/MessageList.tsx | 89 +
.../v2/pages/assistant/components/ModelPicker.tsx | 110 +
.../v2/pages/assistant/components/ReconAIMark.tsx | 55 +
.../src/v2/pages/capacity/capacity.less | 18 +
.../src/v2/pages/capacity/capacity.tsx | 98 +-
.../pages/capacity/components/CapacityDetail.tsx | 49 +-
.../capacity/components/CapacityDetailError.tsx | 35 +-
.../src/v2/pages/containers/containers.tsx | 165 +-
.../src/v2/pages/insights/omInsights.tsx | 2 +-
.../v2/pages/namespaceUsage/namespaceUsage.less | 17 +-
.../src/v2/pages/namespaceUsage/namespaceUsage.tsx | 184 +-
.../recon/ozone-recon-web/src/v2/routes-v2.tsx | 5 +
.../ozone-recon-web/src/v2/types/chatbot.types.ts | 43 +-
.../src/v2/types/container.types.ts | 22 +-
.../hadoop/ozone/recon/TestTarExtractor.java | 131 +
.../hadoop/ozone/recon/api/TestBlocksEndPoint.java | 2 +-
.../ozone/recon/api/TestContainerEndpoint.java | 293 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 14 +-
.../ozone/recon/api/TestOpenContainerCount.java | 2 +-
.../recon/api/TestPendingDeletionEndpoint.java | 35 +-
.../recon/api/TestStorageDistributionEndpoint.java | 64 +-
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 100 +-
.../ozone/recon/api/filters/TestAdminFilter.java | 68 +-
.../agent/TestChatbotAgentExecutionPolicy.java | 221 +
.../agent/TestChatbotAgentListKeysPolicy.java | 258 ++
.../agent/TestChatbotAgentMultiToolAndSummary.java | 188 +
.../agent/TestChatbotAgentToolCallParsing.java | 276 ++
.../recon/chatbot/agent/TestChatbotUtils.java | 316 ++
.../recon/chatbot/api/TestChatbotEndpoint.java | 416 ++
.../chatbot/llm/TestLangChain4jDispatcher.java | 169 +
.../ozone/recon/chatbot/llm/TestLlmRouting.java | 86 +
.../recon/chatbot/recon/TestReconApiAllowlist.java | 52 +
.../chatbot/recon/TestReconEndpointRouter.java | 185 +
.../chatbot/recon/TestReconQueryExecutor.java | 155 +
.../recon/TestReconQueryExecutorListKeys.java | 91 +
.../chatbot/recon/TestReconResponseUnwrapper.java | 56 +
.../recon/TestReconToolCatalogConsistency.java | 102 +
.../chatbot/security/TestCredentialHelper.java | 129 +
.../metrics/TestReconScmContainerSyncMetrics.java | 99 +
.../TestUnhealthyContainersDerbyPerformance.java | 176 +-
.../scm/AbstractReconContainerManagerTest.java | 36 +-
.../ozone/recon/scm/TestReconContainerManager.java | 121 +-
...TestReconIncrementalContainerReportHandler.java | 76 +-
.../ozone/recon/scm/TestReconNodeManager.java | 5 +-
.../scm/TestReconSCMContainerSyncIntegration.java | 1130 +++++
.../TestReconStorageContainerManagerFacade.java | 218 +
.../scm/TestReconStorageContainerSyncHelper.java | 109 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 442 +-
.../recon/tasks/TestReconTaskControllerImpl.java | 95 +-
hadoop-ozone/s3-secret-store/pom.xml | 4 +-
hadoop-ozone/s3gateway/pom.xml | 11 +-
.../org/apache/hadoop/ozone/audit/S3GAction.java | 5 +
.../org/apache/hadoop/ozone/s3/TracingFilter.java | 49 +-
.../s3/commontypes/DirectoryBucketMetadata.java | 79 +
.../ozone/s3/commontypes/ObjectKeyNameAdapter.java | 8 +-
.../ozone/s3/commontypes/RequestParameters.java | 7 +-
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 7 +-
.../ozone/s3/endpoint/BucketCrudHandler.java | 5 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 24 +-
.../s3/endpoint/BucketGetLocationHandler.java | 47 +
.../ozone/s3/endpoint/BucketTaggingHandler.java | 152 +
...CompleteMultipartUploadRequestUnmarshaller.java | 17 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 107 +-
.../s3/endpoint/ListDirectoryBucketsResponse.java | 69 +
.../ozone/s3/endpoint/ListObjectResponse.java | 2 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 131 +-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 66 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 128 +
.../ozone/s3/endpoint/S3ConditionalRequest.java | 27 +
.../s3/endpoint/S3ObjectStreamingWriteGuard.java | 55 +
.../ozone/s3/endpoint/S3ObjectWriteGuard.java | 126 +
.../apache/hadoop/ozone/s3/endpoint/S3Tagging.java | 8 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 114 +-
.../ozone/s3/signature/StringToSignProducer.java | 8 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 15 +
.../hadoop/ozone/client/ClientProtocolStub.java | 28 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 85 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 35 +-
.../ozone/s3/endpoint/EndpointTestUtils.java | 100 +-
...bjectDelete.java => TestBucketGetLocation.java} | 45 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 121 +
...aggingGet.java => TestBucketTaggingDelete.java} | 76 +-
...ctTaggingGet.java => TestBucketTaggingGet.java} | 66 +-
.../ozone/s3/endpoint/TestBucketTaggingPut.java | 206 +
...CompleteMultipartUploadRequestUnmarshaller.java | 63 +
.../s3/endpoint/TestListDirectoryBuckets.java | 187 +
.../s3/endpoint/TestMultipartUploadComplete.java | 35 +
.../hadoop/ozone/s3/endpoint/TestObjectDelete.java | 76 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 53 +-
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 56 +
.../ozone/s3/endpoint/TestObjectMultiDelete.java | 39 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 116 +-
.../ozone/s3/endpoint/TestObjectTaggingGet.java | 27 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 90 +-
.../ozone/s3/endpoint/TestS3ObjectWriteGuard.java | 218 +
.../ozone/s3/endpoint/TestUploadWithStream.java | 30 +
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 81 +-
.../s3/signature/TestStringToSignProducer.java | 28 +
.../hadoop/ozone/s3/signature}/package-info.java | 5 +-
hadoop-ozone/tools/pom.xml | 14 +-
.../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 11 +-
.../hadoop/ozone/local/LocalOzoneCluster.java | 495 ++
.../ozone/local/LocalOzoneClusterConfig.java | 327 ++
.../hadoop/ozone/local/LocalOzoneRuntime.java | 44 +-
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 234 +-
.../apache/hadoop/ozone/local/package-info.java | 2 +-
.../org/apache/hadoop/ozone/shell/OzoneRatis.java | 5 +-
.../apache/hadoop/ozone/utils/AutoCompletion.java | 6 +-
.../hadoop/ozone/local/TestLocalOzoneCluster.java | 356 ++
.../ozone/local/TestLocalOzoneClusterConfig.java | 135 +
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 273 +-
hadoop-ozone/vapor/pom.xml | 8 +-
.../ozone/freon/ClosedContainerReplicator.java | 270 --
.../hadoop/ozone/freon/DatanodeSimulator.java | 21 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 16 +-
pom.xml | 213 +-
1246 files changed, 71602 insertions(+), 21792 deletions(-)
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 56f09137788,3c8714d8e39..110d0e463d6
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@@ -1215,39 -1214,33 +1215,66 @@@ public class OzoneBucket extends WithMe
proxy.deleteObjectTagging(volumeName, name, keyName);
}
+ /**
+ * Gets the lifecycle configuration information.
+ * @return OzoneLifecycleConfiguration or exception is thrown.
+ * @throws IOException
+ */
+ @JsonIgnore
+ public OzoneLifecycleConfiguration getLifecycleConfiguration()
+ throws IOException {
+ return proxy.getLifecycleConfiguration(volumeName, name);
+ }
+
+ /**
+ * Sets the lifecycle configuration for this bucket.
+ * This operation will completely overwrite any existing lifecycle configuration on the bucket.
+ * If the bucket already has a lifecycle configuration, it will be replaced with the new one.
+ *
+ * @param lifecycleConfiguration - lifecycle configuration info to be set.
+ * @throws IOException if there is an error setting the lifecycle configuration.
+ */
+ public void setLifecycleConfiguration(OmLifecycleConfiguration lifecycleConfiguration)
+ throws IOException {
+ proxy.setLifecycleConfiguration(lifecycleConfiguration);
+ }
+
+ /**
+ * Deletes existing lifecycle configuration.
+ * @throws IOException
+ */
+ public void deleteLifecycleConfiguration()
+ throws IOException {
+ proxy.deleteLifecycleConfiguration(volumeName, name);
+ }
+
+ /**
+ * Gets the bucketTags for this bucket.
+ * @return Tags for this bucket.
+ * @throws IOException
+ */
+ @JsonIgnore
+ public Map<String, String> getBucketTagging() throws IOException {
+ return proxy.getBucketTagging(volumeName, name);
+ }
+
+ /**
+ * Sets bucketTags on this bucket (replaces existing tag set).
+ * @param tags Tags to set on the bucket.
+ * @throws IOException
+ */
+ public void putBucketTagging(Map<String, String> tags) throws IOException {
+ proxy.putBucketTagging(volumeName, name, tags);
+ }
+
+ /**
+ * Removes all bucketTags from this bucket.
+ * @throws IOException
+ */
+ public void deleteBucketTagging() throws IOException {
+ proxy.deleteBucketTagging(volumeName, name);
+ }
+
public void setSourcePathExist(boolean b) {
this.sourcePathExist = b;
}
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 20b6379b6bc,518a9b772da..53df47dd2f7
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@@ -1497,33 -1507,31 +1509,60 @@@ public interface ClientProtocol
void deleteObjectTagging(String volumeName, String bucketName, String keyName)
throws IOException;
+ /**
+ * Gets the lifecycle configuration information.
+ * @param volumeName - Volume name.
+ * @param bucketName - Bucket name.
+ * @return OzoneLifecycleConfiguration or exception is thrown.
+ * @throws IOException
+ */
+ OzoneLifecycleConfiguration getLifecycleConfiguration(String volumeName, String bucketName)
+ throws IOException;
+
+ /**
+ * Creates a new lifecycle configuration.
+ * This operation will completely overwrite any existing lifecycle configuration on the bucket.
+ * If the bucket already has a lifecycle configuration, it will be replaced with the new one.
+ * @param lifecycleConfiguration - lifecycle configuration info.
+ * @throws IOException
+ */
+ void setLifecycleConfiguration(OmLifecycleConfiguration lifecycleConfiguration)
+ throws IOException;
+
+ /**
+ * Deletes existing lifecycle configuration.
+ * @param volumeName - Volume name.
+ * @param bucketName - Bucket name.
+ * @throws IOException
+ */
+ void deleteLifecycleConfiguration(String volumeName, String bucketName)
+ throws IOException;
+
+ /**
+ * Gets the tags for an existing bucket.
+ * @param volumeName Volume name.
+ * @param bucketName Bucket name.
+ * @return Tags for the specified bucket.
+ * @throws IOException
+ */
+ Map<String, String> getBucketTagging(String volumeName, String bucketName)
+ throws IOException;
+
+ /**
+ * Sets tags on an existing bucket (replaces existing tag set).
+ * @param volumeName Volume name.
+ * @param bucketName Bucket name.
+ * @param tags Tags to set on the bucket.
+ * @throws IOException
+ */
+ void putBucketTagging(String volumeName, String bucketName,
+ Map<String, String> tags) throws IOException;
+
+ /**
+ * Removes all tags from the specified bucket.
+ * @param volumeName Volume name.
+ * @param bucketName Bucket name.
+ * @throws IOException
+ */
+ void deleteBucketTagging(String volumeName, String bucketName) throws IOException;
}
diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 95bbc8ee49e,57d358e393e..8f2e14fef6f
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@@ -2896,38 -2894,55 +2896,87 @@@ public class RpcClient implements Clien
ozoneManagerClient.deleteObjectTagging(keyArgs);
}
+ @Override
+ public OzoneLifecycleConfiguration getLifecycleConfiguration(String volumeName, String bucketName)
+ throws IOException {
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
+
+ OmLifecycleConfiguration lifecycleConfiguration =
+ ozoneManagerClient.getLifecycleConfiguration(volumeName, bucketName);
+ return OzoneLifecycleConfiguration.fromOmLifecycleConfiguration(
+ lifecycleConfiguration);
+ }
+
+ @Override
+ public void setLifecycleConfiguration(OmLifecycleConfiguration lifecycleConfiguration) throws IOException {
+ Objects.requireNonNull(lifecycleConfiguration, "lifecycleConfiguration == null");
+ verifyVolumeName(lifecycleConfiguration.getVolume());
+ verifyBucketName(lifecycleConfiguration.getBucket());
+
+ LOG.info("Creating lifecycle configuration for: {}/{}", lifecycleConfiguration.getVolume(),
+ lifecycleConfiguration.getBucket());
+ ozoneManagerClient.setLifecycleConfiguration(lifecycleConfiguration);
+ }
+
+ @Override
+ public void deleteLifecycleConfiguration(String volumeName, String bucketName) throws IOException {
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
+
+ LOG.info("Deleting lifecycle Configuration for : {}/{}", volumeName, bucketName);
+ ozoneManagerClient.deleteLifecycleConfiguration(volumeName, bucketName);
+ }
+
+ @Override
+ public Map<String, String> getBucketTagging(String volumeName, String bucketName)
+ throws IOException {
+ if (omVersion.compareTo(OzoneManagerVersion.S3_BUCKET_TAGGING_API) < 0) {
+ throw new IOException("OzoneManager does not support S3 bucket tagging API");
+ }
+
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
+ OmBucketArgs bucketArgs = new OmBucketArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .build();
+ return ozoneManagerClient.getBucketTagging(bucketArgs);
+ }
+
+ @Override
+ public void putBucketTagging(String volumeName, String bucketName,
+ Map<String, String> tags) throws IOException {
+ if (omVersion.compareTo(OzoneManagerVersion.S3_BUCKET_TAGGING_API) < 0) {
+ throw new IOException("OzoneManager does not support S3 bucket tagging API");
+ }
+
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
+ OmBucketArgs bucketArgs = new OmBucketArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .addAllTags(tags)
+ .build();
+ ozoneManagerClient.putBucketTagging(bucketArgs);
+ }
+
+ @Override
+ public void deleteBucketTagging(String volumeName, String bucketName)
+ throws IOException {
+ if (omVersion.compareTo(OzoneManagerVersion.S3_BUCKET_TAGGING_API) < 0) {
+ throw new IOException("OzoneManager does not support S3 bucket tagging API");
+ }
+
+ verifyVolumeName(volumeName);
+ verifyBucketName(bucketName);
+ OmBucketArgs bucketArgs = new OmBucketArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .build();
+ ozoneManagerClient.deleteBucketTagging(bucketArgs);
+ }
+
private static ExecutorService createThreadPoolExecutor(
int corePoolSize, int maximumPoolSize, String threadNameFormat) {
return new ThreadPoolExecutor(corePoolSize, maximumPoolSize,
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 575b8d2aec5,c38163f8753..08b1d8af5a8
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@@ -259,10 -259,10 +259,12 @@@ public final class OmUtils
// keeping it here for compatibility
case GetSnapshotInfo:
case GetObjectTagging:
+ case GetBucketTagging:
+ return true;
case GetQuotaRepairStatus:
case StartQuotaRepair:
+ case GetLifecycleConfiguration:
+ case GetLifecycleServiceStatus:
return true;
case CreateVolume:
case SetVolumeProperty:
@@@ -324,10 -324,9 +326,13 @@@
case QuotaRepair:
case PutObjectTagging:
case DeleteObjectTagging:
+ case PutBucketTagging:
+ case DeleteBucketTagging:
+ case SetLifecycleConfiguration:
+ case DeleteLifecycleConfiguration:
+ case SetLifecycleServiceStatus:
+ case SaveLifecycleScanState:
+ return false;
case UnknownCommand:
return false;
case EchoRPC:
@@@ -382,9 -381,9 +387,11 @@@
case GetKeyInfo:
case GetSnapshotInfo:
case GetObjectTagging:
+ case GetLifecycleConfiguration:
+ case GetLifecycleServiceStatus:
return true;
+ case GetBucketTagging:
+ return true;
case CreateVolume:
case SetVolumeProperty:
case DeleteVolume:
diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index ddb18739012,9de50cd0d35..1b7b368a15a
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@@ -2755,101 -2759,67 +2766,162 @@@ public final class OzoneManagerProtocol
handleError(omResponse);
}
+ @Override
+ public OmLifecycleConfiguration getLifecycleConfiguration(String volumeName,
+ String bucketName) throws IOException {
+ GetLifecycleConfigurationRequest.Builder req =
+ GetLifecycleConfigurationRequest.newBuilder();
+ req.setVolumeName(volumeName);
+ req.setBucketName(bucketName);
+
+ OMRequest omRequest = createOMRequest(Type.GetLifecycleConfiguration)
+ .setGetLifecycleConfigurationRequest(req)
+ .build();
+
+ GetLifecycleConfigurationResponse resp = handleError(submitRequest(
+ omRequest)).getGetLifecycleConfigurationResponse();
+
+ return OmLifecycleConfiguration.getFromProtobuf(
+ resp.getLifecycleConfiguration());
+ }
+
+ @Override
+ public GetLifecycleServiceStatusResponse getLifecycleServiceStatus() throws IOException {
+ OzoneManagerProtocolProtos.GetLifecycleServiceStatusRequest
+ getLifecycleServiceStatusRequest =
+ OzoneManagerProtocolProtos.GetLifecycleServiceStatusRequest
+ .newBuilder().build();
+
+ OMRequest omRequest = createOMRequest(Type.GetLifecycleServiceStatus)
+ .setGetLifecycleServiceStatusRequest(getLifecycleServiceStatusRequest)
+ .build();
+
+ return handleError(submitRequest(omRequest))
+ .getGetLifecycleServiceStatusResponse();
+ }
+
+ @Override
+ public void setLifecycleConfiguration(
+ OmLifecycleConfiguration omLifecycleConfiguration) throws IOException {
+ SetLifecycleConfigurationRequest.Builder req =
+ SetLifecycleConfigurationRequest.newBuilder();
+ LifecycleConfiguration lifecycleConfiguration =
+ omLifecycleConfiguration.getProtobuf();
+ req.setLifecycleConfiguration(lifecycleConfiguration);
+
+ OMRequest omRequest =
+ createOMRequest(Type.SetLifecycleConfiguration)
+ .setSetLifecycleConfigurationRequest(req)
+ .build();
+
+ OMResponse omResponse = submitRequest(omRequest);
+ handleError(omResponse);
+ }
+
+ @Override
+ public void deleteLifecycleConfiguration(String volumeName, String bucketName)
+ throws IOException {
+ DeleteLifecycleConfigurationRequest.Builder req =
+ DeleteLifecycleConfigurationRequest.newBuilder();
+ req.setVolumeName(volumeName);
+ req.setBucketName(bucketName);
+
+ OMRequest omRequest = createOMRequest(Type.DeleteLifecycleConfiguration)
+ .setDeleteLifecycleConfigurationRequest(req)
+ .build();
+
+ handleError(submitRequest(omRequest));
+ }
+
+ @Override
+ public Map<String, String> getBucketTagging(OmBucketArgs args) throws IOException {
+ BucketArgs bucketArgs = BucketArgs.newBuilder()
+ .setVolumeName(args.getVolumeName())
+ .setBucketName(args.getBucketName())
+ .build();
+
+ GetBucketTaggingRequest req =
+ GetBucketTaggingRequest.newBuilder()
+ .setBucketArgs(bucketArgs)
+ .build();
+
+ OMRequest omRequest = createOMRequest(Type.GetBucketTagging)
+ .setGetBucketTaggingRequest(req)
+ .build();
+
+ GetBucketTaggingResponse resp =
+ handleError(submitRequest(omRequest)).getGetBucketTaggingResponse();
+
+ return KeyValueUtil.getFromProtobuf(resp.getTagsList());
+ }
+
+ @Override
+ public void putBucketTagging(OmBucketArgs args) throws IOException {
+ BucketArgs bucketArgs = BucketArgs.newBuilder()
+ .setVolumeName(args.getVolumeName())
+ .setBucketName(args.getBucketName())
+ .addAllTags(KeyValueUtil.toProtobuf(args.getTags()))
+ .build();
+
+ PutBucketTaggingRequest req =
+ PutBucketTaggingRequest.newBuilder()
+ .setBucketArgs(bucketArgs)
+ .build();
+
+ OMRequest omRequest = createOMRequest(Type.PutBucketTagging)
+ .setPutBucketTaggingRequest(req)
+ .build();
+
+ handleError(submitRequest(omRequest));
+ }
+
+ @Override
+ public void suspendLifecycleService() throws IOException {
+ OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest
+ setLifecycleServiceStatusRequest =
+ OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest
+ .newBuilder().setSuspend(true).build();
+
+ OMRequest omRequest = createOMRequest(Type.SetLifecycleServiceStatus)
+ .setSetLifecycleServiceStatusRequest(setLifecycleServiceStatusRequest)
+ .build();
+
+ handleError(submitRequest(omRequest));
+ }
+
+ @Override
+ public void resumeLifecycleService() throws IOException {
+ OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest
+ setLifecycleServiceStatusRequest =
+ OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest
+ .newBuilder().setSuspend(false).build();
+
+ OMRequest omRequest = createOMRequest(Type.SetLifecycleServiceStatus)
+ .setSetLifecycleServiceStatusRequest(setLifecycleServiceStatusRequest)
+ .build();
+
+ handleError(submitRequest(omRequest));
+ }
+
+ @Override
+ public void deleteBucketTagging(OmBucketArgs args) throws IOException {
+ BucketArgs bucketArgs = BucketArgs.newBuilder()
+ .setVolumeName(args.getVolumeName())
+ .setBucketName(args.getBucketName())
+ .build();
+
+ DeleteBucketTaggingRequest req =
+ DeleteBucketTaggingRequest.newBuilder()
+ .setBucketArgs(bucketArgs)
+ .build();
+
+ OMRequest omRequest = createOMRequest(Type.DeleteBucketTagging)
+ .setDeleteBucketTaggingRequest(req)
+ .build();
+
+ handleError(submitRequest(omRequest));
+ }
+
private SafeMode toProtoBuf(SafeModeAction action) {
switch (action) {
case ENTER:
diff --cc hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index 60100d2b5ae,c31e3154c32..03694c17965
--- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@@ -35,20 -35,23 +35,26 @@@ import com.amazonaws.AmazonServiceExcep
import com.amazonaws.AmazonServiceException.ErrorType;
import com.amazonaws.HttpMethod;
import com.amazonaws.services.s3.AmazonS3;
+ import com.amazonaws.services.s3.Headers;
+import com.amazonaws.services.s3.model.AbortIncompleteMultipartUpload;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.Bucket;
+import com.amazonaws.services.s3.model.BucketLifecycleConfiguration;
+ import com.amazonaws.services.s3.model.BucketTaggingConfiguration;
import com.amazonaws.services.s3.model.CanonicalGrantee;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.CreateBucketRequest;
+ import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
+import com.amazonaws.services.s3.model.GetBucketLifecycleConfigurationRequest;
+ import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
+ import com.amazonaws.services.s3.model.GetObjectTaggingRequest;
+ import com.amazonaws.services.s3.model.GetObjectTaggingResult;
import com.amazonaws.services.s3.model.Grantee;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
@@@ -72,9 -75,11 +78,12 @@@ import com.amazonaws.services.s3.model.
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.SetBucketLifecycleConfigurationRequest;
+ import com.amazonaws.services.s3.model.SetBucketTaggingConfigurationRequest;
import com.amazonaws.services.s3.model.SetObjectAclRequest;
+ import com.amazonaws.services.s3.model.SetObjectTaggingRequest;
import com.amazonaws.services.s3.model.Tag;
+ import com.amazonaws.services.s3.model.TagSet;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.UploadPartResult;
import com.amazonaws.services.s3.transfer.TransferManager;
diff --cc hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index b6ae223f295,53cb48e99e9..4a66a428511
--- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@@ -71,10 -72,7 +73,11 @@@ import org.apache.hadoop.ozone.client.O
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
+import org.apache.hadoop.ozone.om.service.KeyLifecycleService;
import org.apache.hadoop.ozone.s3.S3ClientFactory;
import org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils;
import org.apache.hadoop.ozone.s3.endpoint.S3Owner;
@@@ -108,9 -105,8 +111,10 @@@ import software.amazon.awssdk.http.apac
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.model.AbortIncompleteMultipartUpload;
import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+ import software.amazon.awssdk.services.s3.model.Bucket;
+import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
@@@ -124,9 -121,9 +129,11 @@@ import software.amazon.awssdk.services.
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectTaggingRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ExpirationStatus;
import software.amazon.awssdk.services.s3.model.GetBucketAclRequest;
+import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationResponse;
+ import software.amazon.awssdk.services.s3.model.GetBucketTaggingRequest;
+ import software.amazon.awssdk.services.s3.model.GetBucketTaggingResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest;
@@@ -133,12 -131,10 +141,14 @@@ import software.amazon.awssdk.services.
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.LifecycleRule;
+import software.amazon.awssdk.services.s3.model.LifecycleRuleAndOperator;
+import software.amazon.awssdk.services.s3.model.LifecycleRuleFilter;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
+ import software.amazon.awssdk.services.s3.model.ListDirectoryBucketsRequest;
+ import software.amazon.awssdk.services.s3.model.ListDirectoryBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index f2b21a06ad3,f50d812e51f..38b56fba72d
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@@ -157,12 -157,11 +157,16 @@@ enum Type
GetObjectTagging = 141;
DeleteObjectTagging = 142;
SubmitSnapshotDiff = 143;
- SetLifecycleConfiguration = 144;
- GetLifecycleConfiguration = 145;
- DeleteLifecycleConfiguration = 146;
- GetLifecycleServiceStatus = 147;
- SetLifecycleServiceStatus = 148;
- SaveLifecycleScanState = 149;
-
+ // TODO(HDDS-15497): S3 bucket tagging RPCs; handled by OM in a follow-up PR (S3G maps to PUT/GET/DELETE ?tagging).
+ PutBucketTagging = 144;
+ GetBucketTagging = 145;
+ DeleteBucketTagging = 146;
++ SetLifecycleConfiguration = 147;
++ GetLifecycleConfiguration = 148;
++ DeleteLifecycleConfiguration = 149;
++ GetLifecycleServiceStatus = 150;
++ SetLifecycleServiceStatus = 151;
++ SaveLifecycleScanState = 152;
}
enum SafeMode {
@@@ -313,14 -312,15 +317,19 @@@ message OMRequest
optional PutObjectTaggingRequest putObjectTaggingRequest = 141;
optional DeleteObjectTaggingRequest deleteObjectTaggingRequest = 142;
repeated SetSnapshotPropertyRequest SetSnapshotPropertyRequests = 143;
-
- optional SubmitSnapshotDiffRequest submitSnapshotDiffRequest = 144;
-
+ optional SubmitSnapshotDiffRequest submitSnapshotDiffRequest = 144;
-
- optional SetLifecycleConfigurationRequest setLifecycleConfigurationRequest = 145;
- optional GetLifecycleConfigurationRequest getLifecycleConfigurationRequest = 146;
- optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 147;
- optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 148;
- optional SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest = 149;
- optional SaveLifecycleScanStateRequest saveLifecycleScanStateRequest = 150;
+ // TODO: PutBucketTagging — tags in bucketArgs.tags; OM persists to BucketInfo.tags.
+ optional PutBucketTaggingRequest putBucketTaggingRequest = 145;
+ // TODO: GetBucketTagging — volume/bucket in bucketArgs; response returns tag list.
+ optional GetBucketTaggingRequest getBucketTaggingRequest = 146;
+ // TODO: DeleteBucketTagging — clears tags on target bucket (link resolves in OM).
+ optional DeleteBucketTaggingRequest deleteBucketTaggingRequest = 147;
++ optional SetLifecycleConfigurationRequest setLifecycleConfigurationRequest = 148;
++ optional GetLifecycleConfigurationRequest getLifecycleConfigurationRequest = 149;
++ optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 150;
++ optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 151;
++ optional SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest = 152;
++ optional SaveLifecycleScanStateRequest saveLifecycleScanStateRequest = 153;
}
message OMResponse {
@@@ -454,14 -454,15 +463,21 @@@
optional GetObjectTaggingResponse getObjectTaggingResponse = 140;
optional PutObjectTaggingResponse putObjectTaggingResponse = 141;
optional DeleteObjectTaggingResponse deleteObjectTaggingResponse = 142;
-
optional SubmitSnapshotDiffResponse submitSnapshotDiffResponse = 143;
- optional SetLifecycleConfigurationResponse setLifecycleConfigurationResponse = 144;
- optional GetLifecycleConfigurationResponse getLifecycleConfigurationResponse = 145;
- optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 146;
- optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 147;
- optional SetLifecycleServiceStatusResponse setLifecycleServiceStatusResponse = 148;
- optional SaveLifecycleScanStateResponse saveLifecycleScanStateResponse = 149;
+ // TODO: Empty ack after OM applies tag set to OmBucketInfo.
+ optional PutBucketTaggingResponse putBucketTaggingResponse = 144;
+ // TODO: Tag list for S3G GetBucketTagging XML; empty if no tags.
+ optional GetBucketTaggingResponse getBucketTaggingResponse = 145;
+ // TODO: Empty ack after OM clears BucketInfo.tags.
+ optional DeleteBucketTaggingResponse deleteBucketTaggingResponse = 146;
++
++ optional SetLifecycleConfigurationResponse setLifecycleConfigurationResponse = 147;
++ optional GetLifecycleConfigurationResponse getLifecycleConfigurationResponse = 148;
++ optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 149;
++ optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 150;
++ optional SetLifecycleServiceStatusResponse setLifecycleServiceStatusResponse = 151;
++ optional SaveLifecycleScanStateResponse saveLifecycleScanStateResponse = 152;
}
enum Status {
@@@ -2612,38 -2502,32 +2634,67 @@@ service OzoneManagerService
returns(OMResponse);
}
+message GetLifecycleServiceStatusRequest {
+}
+
+message GetLifecycleServiceStatusResponse {
+ required bool isEnabled = 1;
+ optional bool isSuspended = 2;
+ repeated string runningBuckets = 3;
+}
+
+message SetLifecycleServiceStatusRequest {
+ required bool suspend = 1;
+}
+
+message SetLifecycleServiceStatusResponse {
+}
+
-
+message LifecycleScanState {
+ optional string bucketKey = 1; // volume/bucket
+ optional uint64 bucketObjID = 2;
+ optional uint64 lifecycleConfigurationUpdateID = 3;
+ optional uint64 scanStartTime = 4;
+ optional uint64 scanEndTime = 5;
+ optional string lastScannedKey = 6;
+ optional string lastScannedDir = 7;
+ optional string lastScannedDirKey = 8;
+ optional string lastScannedMpuKey = 9;
+}
+
+message SaveLifecycleScanStateRequest {
+ optional LifecycleScanState state = 1;
+}
+
+message SaveLifecycleScanStateResponse {
+}
++
+ // TODO: S3 PutBucketTagging — bucketArgs identifies bucket; tags in bucketArgs.tags replace existing set.
+ message PutBucketTaggingRequest {
+ required BucketArgs bucketArgs = 1;
+ optional uint64 modificationTime = 2;
+ }
+
+ // TODO: Success response; no body (tags stored on OmBucketInfo).
+ message PutBucketTaggingResponse {
+ }
+
+ // TODO: S3 GetBucketTagging — bucketArgs.volumeName/bucketName; link resolved in OM reader.
+ message GetBucketTaggingRequest {
+ required BucketArgs bucketArgs = 1;
+ }
+
+ // TODO: Returns current bucket tags for S3G Tagging XML response.
+ message GetBucketTaggingResponse {
+ repeated hadoop.hdds.KeyValue tags = 1;
+ }
+
+ // TODO: S3 DeleteBucketTagging — clears all tags on bucket (link → source bucket in OM).
+ message DeleteBucketTaggingRequest {
+ required BucketArgs bucketArgs = 1;
+ optional uint64 modificationTime = 2;
+ }
+
+ // TODO: Success response; bucket has no tags after commit.
+ message DeleteBucketTaggingResponse {
+ }
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java
index 8ad5b3ff74b,ed742bad164..0a01f2e493d
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java
@@@ -95,7 -95,9 +95,10 @@@ public final class OMAuditLogger
CMD_AUDIT_ACTION_MAP.put(Type.GetObjectTagging, OMAction.GET_OBJECT_TAGGING);
CMD_AUDIT_ACTION_MAP.put(Type.PutObjectTagging, OMAction.PUT_OBJECT_TAGGING);
CMD_AUDIT_ACTION_MAP.put(Type.DeleteObjectTagging, OMAction.DELETE_OBJECT_TAGGING);
+ CMD_AUDIT_ACTION_MAP.put(Type.GetBucketTagging, OMAction.GET_BUCKET_TAGGING);
+ CMD_AUDIT_ACTION_MAP.put(Type.PutBucketTagging, OMAction.PUT_BUCKET_TAGGING);
+ CMD_AUDIT_ACTION_MAP.put(Type.DeleteBucketTagging, OMAction.DELETE_BUCKET_TAGGING);
+ CMD_AUDIT_ACTION_MAP.put(Type.SetLifecycleServiceStatus, OMAction.SET_LIFECYCLE_SERVICE_STATUS);
}
private static OMAction getAction(OzoneManagerProtocolProtos.OMRequest request) {
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index 04dc20b0afc,0a46c589af2..94a328a77c1
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@@ -346,14 -344,10 +348,18 @@@ public final class OzoneManagerRatisUti
volumeName = keyArgs.getVolumeName();
bucketName = keyArgs.getBucketName();
break;
+ case SetLifecycleConfiguration:
+ return new OMLifecycleConfigurationSetRequest(omRequest);
+ case DeleteLifecycleConfiguration:
+ return new OMLifecycleConfigurationDeleteRequest(omRequest);
+ case SetLifecycleServiceStatus:
+ return new OMLifecycleSetServiceStatusRequest(omRequest);
+ case SaveLifecycleScanState:
+ return new OMLifecycleSaveScanStateRequest(omRequest);
+ case PutBucketTagging:
+ return new S3PutBucketTaggingRequest(omRequest);
+ case DeleteBucketTagging:
+ return new S3DeleteBucketTaggingRequest(omRequest);
default:
throw new OMException("Unrecognized write command type request "
+ cmdType, OMException.ResultCodes.INVALID_REQUEST);
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
index a3df126fdb0,00000000000..ead8acc7cd5
mode 100644,000000..100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
@@@ -1,2131 -1,0 +1,2131 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.service;
+
+import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
+import static org.apache.hadoop.fs.ozone.OzoneTrashPolicy.CURRENT;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT;
+import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.ServiceException;
+import jakarta.annotation.Nullable;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.security.PrivilegedExceptionAction;
+import java.time.Instant;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult;
+import org.apache.hadoop.hdds.utils.FaultInjector;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.ClientVersion;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.KeyManager;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.OzoneTrash;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmLCFilter;
+import org.apache.hadoop.ozone.om.helpers.OmLCRule;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
+import org.apache.hadoop.ozone.om.request.OMClientRequest;
+import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateDirectoryRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyError;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RequestSource;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+import org.apache.ratis.protocol.ClientId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is the background service to manage object lifecycle based on bucket lifecycle configuration.
+ */
+public class KeyLifecycleService extends BackgroundService {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(KeyLifecycleService.class);
+
+ private final OzoneManager ozoneManager;
+ private int keyDeleteBatchSize;
+ private int listMaxSize;
+ private int mpuAbortLimitPerTask;
+ private long cachedDirMaxCount;
+ private final AtomicBoolean suspended;
+ private final AtomicBoolean isServiceEnabled;
+ private final AtomicBoolean moveToTrashEnabled;
+ private KeyLifecycleServiceMetrics metrics;
+ // A set of bucket name that have LifecycleActionTask scheduled
+ private final ConcurrentHashMap<String, LifecycleActionTask> inFlight;
+ private OMMetadataManager omMetadataManager;
+ private int ratisByteLimit;
+ private long stateSaveIntervalMs;
+ private long maxKeysProcessedPerState;
+ private ClientId clientId = ClientId.randomId();
+ private AtomicLong callId = new AtomicLong(0);
+ private OzoneTrash ozoneTrash;
+ private static List<FaultInjector> injectors;
+ private static boolean test = false;
+ private static List<RuleListWithDirectoryList> consolidatedRuleList;
+
+ public KeyLifecycleService(OzoneManager ozoneManager,
+ KeyManager manager, long serviceInterval,
+ long serviceTimeout, int poolSize,
+ ConfigurationSource conf) {
+ super(KeyLifecycleService.class.getSimpleName(), serviceInterval, TimeUnit.MILLISECONDS,
+ poolSize, serviceTimeout, ozoneManager.getThreadNamePrefix());
+ this.ozoneManager = ozoneManager;
+ this.keyDeleteBatchSize = conf.getInt(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE,
+ OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE_DEFAULT);
+ Preconditions.checkArgument(keyDeleteBatchSize > 0,
+ OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE + " should be a positive value.");
+ this.listMaxSize = keyDeleteBatchSize >= 10000 ? keyDeleteBatchSize : 10000;
+ this.mpuAbortLimitPerTask = conf.getInt(OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK,
+ OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK_DEFAULT);
+ Preconditions.checkArgument(mpuAbortLimitPerTask > 0,
+ OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK + " should be a positive value.");
+ this.cachedDirMaxCount = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT,
+ OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT);
+ this.suspended = new AtomicBoolean(false);
+ this.metrics = KeyLifecycleServiceMetrics.create();
+ this.isServiceEnabled = new AtomicBoolean(conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED,
+ OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT));
+ this.moveToTrashEnabled = new AtomicBoolean(conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED,
+ OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT));
+ this.stateSaveIntervalMs = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT);
+ if (!test && stateSaveIntervalMs <= 0) {
+ LOG.warn("Illegal value {} for Property {}. Set {} to {}", stateSaveIntervalMs,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT);
+ stateSaveIntervalMs = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT;
+ }
+ this.maxKeysProcessedPerState = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT);
+ if (!test && maxKeysProcessedPerState <= 0) {
+ LOG.warn("Illegal value {} for Property {}. Set {} to {}", maxKeysProcessedPerState,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT);
+ maxKeysProcessedPerState = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT;
+ }
+ LOG.info("stateSaveIntervalMs = {}, maxKeysProcessedPerState = {}", stateSaveIntervalMs, maxKeysProcessedPerState);
+ this.inFlight = new ConcurrentHashMap();
+ this.omMetadataManager = ozoneManager.getMetadataManager();
+ int limit = (int) conf.getStorageSize(
+ OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT,
+ OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT,
+ StorageUnit.BYTES);
+ // always go to 90% of max limit for request as other header will be added
+ this.ratisByteLimit = (int) (limit * 0.9);
+ this.ozoneTrash = ozoneManager.getOzoneTrash();
+ }
+
+ @Override
+ public BackgroundTaskQueue getTasks() {
+ BackgroundTaskQueue queue = new BackgroundTaskQueue();
+ if (!shouldRun()) {
+ return queue;
+ }
+
+ List<OmLifecycleConfiguration> lifecycleConfigurationList = null;
+ try {
+ lifecycleConfigurationList = omMetadataManager.listLifecycleConfigurations();
+ } catch (OMException e) {
+ LOG.error("Failed to list lifecycle configurations", e);
+ return queue;
+ }
+ for (OmLifecycleConfiguration lifecycleConfiguration : lifecycleConfigurationList) {
+ try {
+ lifecycleConfiguration.valid();
+ } catch (OMException e) {
+ LOG.error("Skip invalid lifecycle configuration for {}/{}: LifecycleConfiguration:\n {}",
+ lifecycleConfiguration.getVolume(), lifecycleConfiguration.getBucket(),
+ lifecycleConfiguration.getProtobuf(), e);
+ continue;
+ }
+ String bucketKey = omMetadataManager.getBucketKey(lifecycleConfiguration.getVolume(),
+ lifecycleConfiguration.getBucket());
+ if (lifecycleConfiguration.getRules().stream().anyMatch(r -> r.isEnabled())) {
+ LifecycleActionTask task = new LifecycleActionTask(lifecycleConfiguration);
+ if (this.inFlight.putIfAbsent(bucketKey, task) == null) {
+ queue.add(task);
+ LOG.info("LifecycleActionTask of {} is scheduled", bucketKey);
+ } else {
+ metrics.incrNumSkippedTask();
+ LOG.info("LifecycleActionTask of {} is already running", bucketKey);
+ }
+ } else {
+ LOG.info("LifecycleConfiguration of {} is not enabled", bucketKey);
+ }
+ }
+ LOG.info("{} LifecycleActionTasks scheduled", queue.size());
+ return queue;
+ }
+
+ private boolean shouldRun() {
+ if (getOzoneManager() == null) {
+ // OzoneManager can be null for testing
+ return true;
+ }
+ return isServiceEnabled.get() && !suspended.get() && getOzoneManager().isLeaderReady();
+ }
+
+ public KeyLifecycleServiceMetrics getMetrics() {
+ return metrics;
+ }
+
+ public OzoneManager getOzoneManager() {
+ return ozoneManager;
+ }
+
+ /**
+ * Suspend the service.
+ */
+ public void suspend() {
+ suspended.set(true);
+ }
+
+ /**
+ * Resume the service if suspended.
+ */
+ public void resume() {
+ suspended.set(false);
+ }
+
+ public boolean isSuspended() {
+ return suspended.get();
+ }
+
+ @Override
+ public void shutdown() {
+ super.shutdown();
+ KeyLifecycleServiceMetrics.unregister();
+ }
+
+ /**
+ * Build a GetLifecycleServiceStatusResponse instance.
+ * @return GetLifecycleServiceStatusResponse instance
+ */
+ public GetLifecycleServiceStatusResponse status() {
+ Set<String> runningBuckets = new HashSet<>(inFlight.keySet());
+ return GetLifecycleServiceStatusResponse.newBuilder()
+ .setIsEnabled(isServiceEnabled.get())
+ .setIsSuspended(suspended.get())
+ .addAllRunningBuckets(runningBuckets)
+ .build();
+ }
+
+ /**
+ * A lifecycle action task for one specific bucket, scanning OM DB and evaluating if any existing
+ * object/key qualified for expiration according to bucket's lifecycle configuration, and sending
+ * key delete command respectively.
+ */
+ public final class LifecycleActionTask implements BackgroundTask {
+ private final OmLifecycleConfiguration policy;
+ private long taskStartTime;
+ private long numKeyIterated = 0;
+ private long numDirIterated = 0;
+ private long numDirDeleted = 0;
+ private long numKeyDeleted = 0;
+ private long sizeKeyDeleted = 0;
+ private long numKeyRenamed = 0;
+ private long sizeKeyRenamed = 0;
+ private long numDirRenamed = 0;
+ private long numMultipartUploadIterated = 0;
+ private long numMultipartUploadAborted = 0;
+ private String lastScannedKey;
+ private String lastScannedDir;
+ private String lastScannedDirKey;
+
+ private long lastStateSaveTime = Time.monotonicNow();
+ private long lastStateSaveKeyCount = 0;
+
+ private boolean shouldSaveState() {
+ if ((Time.monotonicNow() - lastStateSaveTime) > stateSaveIntervalMs ||
+ (numKeyIterated - lastStateSaveKeyCount) >= maxKeysProcessedPerState) {
+ return true;
+ }
+ return false;
+ }
+
+ public LifecycleActionTask(OmLifecycleConfiguration lcConfig) {
+ this.policy = lcConfig;
+ }
+
+ @Override
+ public int getPriority() {
+ return 0;
+ }
+
+ @Override
+ public BackgroundTaskResult call() {
+ EmptyTaskResult result = EmptyTaskResult.newResult();
+ String bucketKey = omMetadataManager.getBucketKey(policy.getVolume(), policy.getBucket());
+ // Check if this is the Leader OM. If not leader, no need to execute this task.
+ if (shouldRun()) {
+ LOG.info("Running LifecycleActionTask {}", bucketKey);
+ taskStartTime = Time.monotonicNow();
+ lastStateSaveTime = taskStartTime;
+ OmBucketInfo bucket;
+ try {
+ if (getInjector(0) != null) {
+ getInjector(0).pause();
+ }
+ bucket = omMetadataManager.getBucketTable().get(bucketKey);
+ if (bucket == null) {
+ LOG.warn("Bucket {} cannot be found, might be deleted during this task's execution", bucketKey);
+ onFailure(bucketKey);
+ return result;
+ }
+ if (bucket.getObjectID() != policy.getBucketObjectID()) {
+ LOG.warn("Bucket object ID doesn't match. ID in bucket is {}, ID in LifecycleConfiguration is {}.",
+ bucket.getObjectID(), policy.getBucketObjectID());
+ onFailure(bucketKey);
+ return result;
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to get Bucket {}", bucketKey, e);
+ onFailure(bucketKey);
+ return result;
+ }
+
+ OmLifecycleScanState.Builder scanStateBuilder = null;
+ try {
+ OmLifecycleScanState scanState = omMetadataManager.getLifecycleScanStateTable().get(bucketKey);
+ if (scanState == null || (scanState.getBucketObjID() != bucket.getObjectID() ||
+ scanState.getLifecycleConfigurationUpdateID() != policy.getUpdateID() ||
+ scanState.getScanEndTime() != null)) {
+ scanStateBuilder = new OmLifecycleScanState.Builder();
+ scanStateBuilder.setBucketKey(bucketKey);
+ scanStateBuilder.setScanStartTime(System.currentTimeMillis());
+ scanStateBuilder.setBucketObjID(bucket.getObjectID());
+ scanStateBuilder.setLifecycleConfigurationUpdateID(policy.getUpdateID());
+ LOG.info("Create/Recreate OmLifecycleScanState for {} bucket {} bucketID {} " +
+ "lifecycleConfigurationUpdateID {}", bucket.getBucketLayout(), bucketKey, bucket.getObjectID(),
+ policy.getUpdateID());
+ } else {
+ scanStateBuilder = scanState.toBuilder();
+ LOG.info("Resume OmLifecycleScanState {}", scanState);
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to get scan state for bucket {}", bucketKey, e);
+ }
+
+ try {
+ List<OmLCRule> originRuleList = policy.getRules();
+ // remove disabled rules
+ List<OmLCRule> ruleList = originRuleList.stream().filter(
+ r -> r.isEnabled()).collect(Collectors.toList());
+
+ List<OmLCRule> expirationRules = ruleList.stream()
+ .filter(r -> r.getExpiration() != null)
+ .collect(Collectors.toList());
+ List<OmLCRule> mpuRules = ruleList.stream()
+ .filter(r -> r.getAbortIncompleteMultipartUpload() != null)
+ .collect(Collectors.toList());
+
+ if (!expirationRules.isEmpty()) {
+ LimitedExpiredObjectList expiredKeyList = new LimitedExpiredObjectList(listMaxSize);
+ LimitedExpiredObjectList expiredDirList = new LimitedExpiredObjectList(listMaxSize);
+ Table<String, OmKeyInfo> keyTable = omMetadataManager.getKeyTable(bucket.getBucketLayout());
+ /**
+ * Filter treatment.
+ * "" - all objects
+ * "/" - if it's OBS/Legacy, means keys starting with "/"; If it's FSO, not supported
+ * "/key" - if it's OBS/Legacy, means keys starting with "/key", "/" is literally "/";
+ * If it's FSO, means keys or dirs starting with "key", "/" will be treated as separator mark.
+ * "key" - if it's OBS/Legacy, means keys starting with "key";
+ * if it's FSO, means keys for dirs starting with "key" too.
+ * "dir/" - if it's OBS/Legacy, means keys starting with "dir/";
+ * - if it's FSO, means keys/dirs under directory "dir", doesn't include directory "dir" itself.
+ * - For FSO bucket, as directory ModificationTime will not be updated when any of its child
+ * key/subdir changes, so remember to add the tailing slash "/" when configure prefix, otherwise
+ * the whole directory will be expired and deleted once its ModificationTime meats the condition.
+ */
+ if (bucket.getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ OmVolumeArgs volume;
+ try {
+ volume = omMetadataManager.getVolumeTable().get(omMetadataManager.getVolumeKey(bucket.getVolumeName()));
+ if (volume == null) {
+ LOG.warn("Volume {} cannot be found, might be deleted during this task's execution",
+ bucket.getVolumeName());
+ onFailure(bucketKey);
+ return result;
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to get volume {}", bucket.getVolumeName(), e);
+ onFailure(bucketKey);
+ return result;
+ }
+ evaluateFSOBucket(volume, bucket, bucketKey, keyTable, expirationRules, expiredKeyList,
+ expiredDirList, scanStateBuilder);
+ } else {
+ // use bucket name as key iterator prefix
+ evaluateBucket(bucket, keyTable, expirationRules, expiredKeyList, scanStateBuilder);
+ }
+
+ if (expiredKeyList.isEmpty() && expiredDirList.isEmpty()) {
+ LOG.info("No expired keys/dirs found/remained for bucket {}", bucketKey);
+ sendSaveScanStateRequest(scanStateBuilder, true);
+ } else {
+ LOG.info("{} expired keys and {} expired dirs found and remained for bucket {}",
+ expiredKeyList.size(), expiredDirList.size(), bucketKey);
+
+ // If trash is enabled, move files to trash, instead of send delete requests.
+ // OBS bucket doesn't support trash.
+ if (bucket.getBucketLayout() == OBJECT_STORE) {
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), expiredKeyList,
+ false, scanStateBuilder, true);
+ } else {
+ // handle keys first, then directories
+ handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, true);
+ handleAndClearFullList(bucket, expiredDirList, true, scanStateBuilder, true);
+ }
+ }
+ }
+
+ if (!mpuRules.isEmpty()) {
+ processMultipartUploads(bucket, mpuRules);
+ }
+ } catch (Throwable e) {
+ LOG.error("Failed to evaluate lifecycle configuration for bucket {}", bucketKey, e);
+ onFailure(bucketKey);
+ return result;
+ }
+
+ onSuccess(bucketKey);
+ }
+
+ // By design, no one cares about the results of this call back.
+ return result;
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ private void evaluateFSOBucket(OmVolumeArgs volume, OmBucketInfo bucket, String bucketKey,
+ Table<String, OmKeyInfo> keyTable, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList expiredKeyList, LimitedExpiredObjectList expiredDirList,
+ OmLifecycleScanState.Builder scanStateBuilder) {
+ List<OmLCRule> prefixRuleList =
+ ruleList.stream().filter(r -> r.isPrefixEnable()).collect(Collectors.toList());
+ // r.isPrefixEnable() == false means empty filter
+ List<OmLCRule> noPrefixRuleList =
+ ruleList.stream().filter(r -> !r.isPrefixEnable()).collect(Collectors.toList());
+
+ if (!noPrefixRuleList.isEmpty()) {
+ // evaluate all rules against each key
+ prefixRuleList.addAll(noPrefixRuleList);
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, "",
+ prefixRuleList, expiredKeyList, expiredDirList, scanStateBuilder);
+ return;
+ }
+
+ List<RuleListWithDirectoryList> unionPrefixRuleList =
+ getRuleUnion(volume.getObjectID(), bucket, prefixRuleList, bucketKey);
+
+ if (unionPrefixRuleList != null) {
+ if (unionPrefixRuleList.isEmpty()) {
+ // fallback to evaluate the whole bucket
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, "",
+ prefixRuleList, expiredKeyList, expiredDirList, scanStateBuilder);
+ } else {
+ for (RuleListWithDirectoryList ruleWithDirList : unionPrefixRuleList) {
+ List<OmLCRule> rules = ruleWithDirList.getRuleList();
+ DirectoryList dir = ruleWithDirList.getDirList();
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, dir.getLastSubDirPath(),
+ dir.getLastSubDir(), dir.getLastSubDirKey(),
+ rules, expiredKeyList, expiredDirList, scanStateBuilder);
+ }
+ }
+ }
+ }
+
+ /**
+ * Finds the directory union list from a list of prefixes and sorts them
+ * according to the FSO depth-first iteration order.
+ */
+ private List<RuleListWithDirectoryList> getRuleUnion(long volumeId, OmBucketInfo bucket,
+ List<OmLCRule> rules, String bucketKey) {
+
+ if (rules.isEmpty() || rules.stream().anyMatch(
+ r -> r.getEffectivePrefix() == null || r.getEffectivePrefix().isEmpty())) {
+ // The union of anything with the root is just the root itself.
+ return new ArrayList();
+ }
+
+ List<RuleListWithDirectoryList> effectiveRuleList = new ArrayList<>();
+ for (OmLCRule rule : rules) {
+ String prefix = rule.getEffectivePrefix();
+ // Resolve each prefix to actual FSO directories in the DB
+ try {
+ if (!prefix.endsWith(OzoneConsts.OM_KEY_PREFIX)) {
+ // FSO bucket doesn't allow prefix without tailing '/'
+ // Prefix ends with a slash, it explicitly refers to a directory (e.g. "log/")
+ LOG.warn("Skip rule {} since FILE_SYSTEM_OPTIMIZED bucket prefix must end with '/'", rule);
+ continue;
+ }
+
+ // Normalize by removing the trailing slash for uniform comparison
+ String normalizedPrefix = prefix.substring(0, prefix.length() - 1);
+ DirectoryList dirList = getDirList(volumeId, bucket, normalizedPrefix, bucketKey);
+ // If the prefix is log/, and "log" dir really exists, then the matched dir is "log".
+ // Otherwise, this rule doesn't match any dir/file in this FSO bucket, this rule can be skipped.
+ if (!dirList.isEmpty() && dirList.isAllResolvedPrefix()) {
+ RuleListWithDirectoryList ruleListWithDirectoryList = new RuleListWithDirectoryList(
+ Collections.singletonList(rule), dirList, prefix);
+ effectiveRuleList.add(ruleListWithDirectoryList);
+ }
+ } catch (IOException e) {
+ // Directory doesn't exist or IO error, skip this rule
+ LOG.warn("Skip to evaluate rule {} due to failed to resolve prefix {} for bucket {}",
+ rule, prefix, bucketKey, e);
+ }
+ }
+
+ if (effectiveRuleList.isEmpty()) {
+ // there is no valid rule found, either prefix doesn't end with "/",
+ // or any directory along the prefix cannot be found.
+ LOG.warn("Prefix of all rules of bucket {} cannot be resolved to an existing directory. ", bucketKey);
+ return null;
+ }
+
+ if (effectiveRuleList.size() == 1) {
+ return effectiveRuleList;
+ }
+
+ // Find if one rule's prefix is the sub string of another rule's prefix.
+ // e.g.
+ // dir1/dir2/, dir1/dir2/dir3/, dir1/ -> dir1/
+ // dir1/dir2/, dir1/dir3/, dir1/dir4/ -> dir1/dir2/, dir1/dir3/, dir1/dir4dir1/
+ // dir1/dir2/dir3/, dir1/dir2/, dir2/ -> dir1/dir2/, dir2/
+ // dir1/dir2/, dir1/dir3/, dir1/ -> dir1/
+ List<RuleListWithDirectoryList> consolidatedRules = new ArrayList<>();
+ Set<OmLCRule> skipEvaluatedRuleList = new HashSet<>();
+ for (int i = 0; i < effectiveRuleList.size(); i++) {
+ OmLCRule rule = effectiveRuleList.get(i).getRuleList().get(0);
+ if (skipEvaluatedRuleList.contains(rule)) {
+ continue;
+ }
+
+ RuleListWithDirectoryList consolidatedCandidate = new RuleListWithDirectoryList();
+ String consolidatedPrefix = effectiveRuleList.get(i).getConsolidatedPrefix();
+ String finalRuleIndexID = rule.getId();
+ DirectoryList finalDirList = effectiveRuleList.get(i).getDirList();
+ for (int j = i + 1; j < effectiveRuleList.size(); j++) {
+ OmLCRule otherRule = effectiveRuleList.get(j).getRuleList().get(0);
+ if (skipEvaluatedRuleList.contains(otherRule)) {
+ continue;
+ }
+
+ DirectoryList otherDirList = effectiveRuleList.get(j).getDirList();
+ String otherPrefix = otherRule.getEffectivePrefix();
+ if (otherPrefix.startsWith(consolidatedPrefix)) {
+ LOG.info("Rule {}'s prefix {} is sub string of rule {}'s prefix {}. " +
+ " Consolidate {} into {}.", otherRule.getId(), otherPrefix, finalRuleIndexID,
+ consolidatedPrefix, otherRule.getId(), finalRuleIndexID);
+ consolidatedCandidate.addRule(otherRule);
+ skipEvaluatedRuleList.add(otherRule);
+ } else if (consolidatedPrefix.startsWith(otherPrefix)) {
+ LOG.info("Rule {}'s prefix {} is sub string of rule {}'s prefix {}. Consolidate {} int {}. ",
+ consolidatedPrefix, consolidatedPrefix, otherRule.getId(), otherPrefix, consolidatedPrefix,
+ otherRule.getId());
+ consolidatedPrefix = otherPrefix;
+ finalRuleIndexID = otherRule.getId();
+ finalDirList = otherDirList;
+ consolidatedCandidate.addRule(otherRule);
+ skipEvaluatedRuleList.add(otherRule);
+ }
+ }
+
+ consolidatedCandidate.addRule(rule);
+ consolidatedCandidate.setDirList(finalDirList);
+ consolidatedCandidate.setConsolidatedPrefix(consolidatedPrefix);
+ consolidatedRules.add(consolidatedCandidate);
+ }
+
+ // Sort the list of paths lexicographically.
+ // FSO Depth-First Search order evaluates directories in lexicographical order
+ // (since it retrieves entries from RocksDB sorted by name within the same parent).
+ // Standard string sort on logical paths separated by "/" perfectly matches this DFS order.
+ List<RuleListWithDirectoryList> sortedConsolidatedRules =
+ consolidatedRules.stream().sorted(new RuleListWithDirectoryListOrder()).collect(Collectors.toList());
+
+ LOG.info("Final consolidated rules: " +
+ sortedConsolidatedRules.stream().map(RuleListWithDirectoryList::toString).collect(Collectors.joining(", ")));
+ if (test) {
+ consolidatedRuleList = sortedConsolidatedRules;
+ }
+ return sortedConsolidatedRules;
+ }
+
+ private boolean canSkipDir(String currentDirPath, String lastScannedDirInState) {
+ if (lastScannedDirInState == null || currentDirPath.isEmpty()) {
+ return false;
+ }
+ String[] cur = currentDirPath.split(OM_KEY_PREFIX);
+ String[] last = lastScannedDirInState.split(OM_KEY_PREFIX);
+ int n = Math.min(cur.length, last.length);
+ for (int i = 0; i < n; i++) {
+ int cmp = cur[i].compareTo(last[i]);
+ if (cmp != 0) {
+ // current name > last name -> skip
+ return cmp > 0;
+ }
+ }
+ return false;
+ }
+
+ @SuppressWarnings({"checkstyle:parameternumber", "checkstyle:MethodLength"})
+ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table<String, OmKeyInfo> keyTable,
+ String directoryPath, @Nullable OmDirectoryInfo dir, String dirKey, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList keyList, LimitedExpiredObjectList dirList,
+ OmLifecycleScanState.Builder scanStateBuilder) {
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ LimitedSizeStack stack = new LimitedSizeStack(cachedDirMaxCount);
+ String lastScannedDirInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDir();
+ String lastScannedDirKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDirKey();
+ String lastScannedKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedKey();
+ try {
+ if (dir != null) {
+ stack.push(new PendingEvaluateDirectory(dir, dirKey, directoryPath, null));
+ } else {
+ // put a placeholder PendingEvaluateDirectory to stack for bucket
+ stack.push(new PendingEvaluateDirectory(null, "", "", null));
+ }
+ } catch (CapacityFullException e) {
+ LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, directoryPath != null ? directoryPath : "", e);
+ return;
+ }
+
+ HashSet<Long> deletedDirSet = new HashSet<>();
+ while (!stack.isEmpty()) {
+ if (!shouldRun()) {
+ LOG.info("LifecycleActionTask for bucket {} stopping. " +
+ "Service enabled: {}, suspended: {}, leader ready: {}",
+ bucketName, isServiceEnabled.get(), suspended.get(),
+ getOzoneManager() != null ? getOzoneManager().isLeaderReady() : "N/A");
+ return;
+ }
+
+ PendingEvaluateDirectory item = stack.pop();
+ OmDirectoryInfo currentDir = item.getDirectoryInfo();
+ String currentDirPath = item.getDirPath();
+ long currentDirObjID = currentDir == null ? bucket.getObjectID() : currentDir.getObjectID();
+ String currentDirTableKey = item.getDirTableKey();
+
+ /**
+ * /
+ * dir1 dir2 dir3 dir30
+ * / \ / \
+ * dir4 dir5 dir6 dir7
+ * / \
+ * dir8 dir9
+ * lastScannedDir = dir3/dir6/dir8, which means
+ * Scanned:
+ * dir30
+ * dir3/dir7
+ * dir3/dir6/dir9
+ * Half scanned:
+ * dir3/dir6/dir8
+ * Not scanned:
+ * dir3/dir6
+ * dir3
+ * dir1/dir5
+ * dir/dir4
+ * dir1
+ * directoryTable table key format : /volumeId/bucketId/parentId/dirName
+ * based on the depth first evaluation order, and stack push posh iteration pattern
+ * - dir1, on grand level of lastScannedDir, and name order < lastScannedDir grand, not scanned
+ * - dir2, on grand level of lastScannedDir, and name order < lastScannedDir grand, not scanned
+ * - dir3, grand of lastScannedDir, not scanned
+ * - dir30, on grand level of lastScannedDir, and name order > lastScannedDir grand, scanned, skip
+ * - dir3/dir6, parent of lastScannedDir, not scanned
+ * - dir3/dir7, parent level of lastScannedDir, and name order > lastScannedDir parent, scanned, skip
+ * - dir3/dir8, lastScannedDir, partially scanned,
+ * - dir3/dir9, same the same parentID as lastScannedDir, and name order > lastScannedDir, scanned, skip
+ */
+ if (canSkipDir(currentDirPath, lastScannedDirInState)) {
+ LOG.info("Skip {} in LifecycleActionTask for bucket {}. ", currentDirPath, bucketName);
+ continue;
+ }
+
+ lastScannedDir = currentDirPath;
+ lastScannedDirKey = currentDirTableKey;
+ if (shouldSaveState()) {
+ flushAndSaveState(bucket, keyList, dirList, scanStateBuilder);
+ }
+
+ // use current directory's object ID to iterate the keys and directories under it
+ String prefix =
+ OM_KEY_PREFIX + volumeObjId + OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX + currentDirObjID;
+ LOG.debug("Prefix {} for {}/{}", prefix, bucket.getVolumeName(), bucket.getBucketName());
+
+ // get direct sub directories
+ DirectoryList subDirSummary;
+ boolean newSubDirPushed = false;
+ long deletedDirCount = 0;
+ if (item.isFirstEvaluate()) {
+ try {
+ subDirSummary = getSubDirectory(currentDirObjID, prefix, omMetadataManager);
+ } catch (IOException e) {
+ // log failure, continue to process other directories in stack
+ LOG.warn("Failed to get sub directories of {} under {}/{}", currentDirPath, volumeName, bucketName, e);
+ continue;
+ }
+ } else {
+ // this item is a parent directory, check how many sub directories are deleted.
+ subDirSummary = item.getSubDirSummary();
+ for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) {
+ if (deletedDirSet.remove(subDir.getObjectID())) {
+ deletedDirCount++;
+ }
+ }
+ }
+
+ if (item.isFirstEvaluate()) {
+ // filter sub directory list
+ if (!subDirSummary.getSubDirList().isEmpty()) {
+ Iterator<OmDirectoryInfo> iterator = subDirSummary.getSubDirList().iterator();
+ while (iterator.hasNext()) {
+ OmDirectoryInfo subDir = iterator.next();
+ String subDirPath = currentDirPath.isEmpty() ? subDir.getName() :
+ currentDirPath + OM_KEY_PREFIX + subDir.getName();
+ if (subDirPath.startsWith(TRASH_PREFIX)) {
+ iterator.remove();
+ continue;
+ }
+ boolean matched = false;
+ for (OmLCRule rule : ruleList) {
+ if (rule.getEffectivePrefix() != null && subDirPath.startsWith(rule.getEffectivePrefix())) {
+ matched = true;
+ break;
+ }
+ }
+ if (!matched) {
+ iterator.remove();
+ }
+ }
+ }
+
+ if (!subDirSummary.getSubDirList().isEmpty()) {
+ item.setDirectoryList(subDirSummary);
+ item.setFirstEvaluate(false);
+ try {
+ stack.push(item);
+ } catch (CapacityFullException e) {
+ LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, currentDirPath, e);
+ return;
+ }
+
+ // depth first evaluation, push subDirs into stack
+ for (int i = 0; i < subDirSummary.getSubDirCount(); i++) {
+ OmDirectoryInfo subDir = subDirSummary.getSubDirList().get(i);
+ String subDirPath = currentDirPath.isEmpty() ? subDir.getName() :
+ currentDirPath + OM_KEY_PREFIX + subDir.getName();
+ try {
+ stack.push(new PendingEvaluateDirectory(subDir, subDirSummary.getSubDirKeyList().get(i),
+ subDirPath, null));
+ } catch (CapacityFullException e) {
+ LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, subDirPath, e);
+ return;
+ }
+ }
+ newSubDirPushed = true;
+ }
+ }
+
+ if (newSubDirPushed) {
+ continue;
+ }
+
+ // evaluate direct files, first check cache, then check table
+ // there are three cases:
+ // a. key is deleted in cache, while it's not deleted in table yet
+ // b. key is new added in cache, not in table yet
+ // c. key is updated in cache(rename), but not updated in table yet
+ // in this case, the fromKey is a deleted key in cache, and the toKey is a newly added key in cache,
+ // and fromKey is also in table
+ long numKeysUnderDir = 0;
+ long numKeysExpired = 0;
+ HashSet<String> deletedKeySetInCache = new HashSet();
+ HashSet<String> keySetInCache = new HashSet();
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter = keyTable.cacheIterator();
+ while (cacheIter.hasNext()) {
+ Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry = cacheIter.next();
+ OmKeyInfo key = entry.getValue().getCacheValue();
+ if (key == null) {
+ deletedKeySetInCache.add(entry.getKey().getCacheKey());
+ continue;
+ }
+ if (key.getParentObjectID() == currentDirObjID) {
+ numKeysUnderDir++;
+ keySetInCache.add(entry.getKey().getCacheKey());
+ String keyPath = currentDirPath.isEmpty() ? key.getKeyName() :
+ currentDirPath + OM_KEY_PREFIX + key.getKeyName();
+ for (OmLCRule rule : ruleList) {
+ if (rule.match(key, keyPath)) {
+ // mark key as expired, check next key
+ if (keyList.isFull()) {
+ // if keyList is full, send delete/rename request for expired keys
+ handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false);
+ }
+ keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID());
+ numKeysExpired++;
+ break;
+ }
+ }
+ lastScannedKey = entry.getKey().getCacheKey();
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyTblItr =
+ keyTable.iterator(prefix)) {
+ boolean seekPerformed = false;
+ if (lastScannedDirKeyInState != null && lastScannedDirKeyInState.compareTo(currentDirTableKey) == 0 &&
+ lastScannedKeyInState != null && lastScannedKeyInState.startsWith(prefix)) {
+ LOG.info("Seek to key {} under directory {}", scanStateBuilder.getLastScannedKey(), lastScannedDirInState);
+ keyTblItr.seek(scanStateBuilder.getLastScannedKey());
+ seekPerformed = true;
+ }
+
+ while (keyTblItr.hasNext()) {
+ if (shouldSaveState()) {
+ LOG.info("Saving scan state for bucket {} at key {}", bucketName, lastScannedKey);
+ flushAndSaveState(bucket, keyList, dirList, scanStateBuilder);
+ }
+ Table.KeyValue<String, OmKeyInfo> keyValue = keyTblItr.next();
+ OmKeyInfo key = keyValue.getValue();
+ String keyPath = currentDirPath.isEmpty() ? key.getKeyName() :
+ currentDirPath + OM_KEY_PREFIX + key.getKeyName();
+ if (seekPerformed && keyValue.getKey().equals(scanStateBuilder.getLastScannedKey())) {
+ continue;
+ }
+ if (deletedKeySetInCache.remove(keyValue.getKey()) || keySetInCache.remove(keyValue.getKey())) {
+ continue;
+ }
+ numKeyIterated++;
+ numKeysUnderDir++;
+ for (OmLCRule rule : ruleList) {
+ if (key.getParentObjectID() == currentDirObjID && rule.match(key, keyPath)) {
+ // mark key as expired, check next key
+ if (keyList.isFull()) {
+ // if keyList is full, send delete request for pending deletion keys
+ handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false);
+ }
+ keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID());
+ numKeysExpired++;
+ break;
+ }
+ }
+ lastScannedKey = keyValue.getKey();
+ }
+ } catch (IOException e) {
+ // log failure and continue the process other directories in stack
+ LOG.warn("Failed to iterate keyTable for bucket {}/{}", volumeName, bucketName, e);
+ continue;
+ }
+
+ // if this directory is empty or all files/subDirs are expired, evaluate itself
+ if ((numKeysUnderDir == 0 && subDirSummary.getSubDirCount() == 0) ||
+ (numKeysUnderDir == numKeysExpired && deletedDirCount == subDirSummary.getSubDirCount())) {
+ List<String> pathList = new ArrayList<>();
+ boolean skipDir = false;
+ for (int i = 0; i < ruleList.size(); i++) { // NOPMD
+ OmLCRule rule = ruleList.get(i);
+ String path = rule.getEffectivePrefix() != null && rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX) ?
+ currentDirPath + OM_KEY_PREFIX : currentDirPath;
+ if (path != null && path.equals(rule.getEffectivePrefix())) {
+ LOG.info("Prefix directory {} doesn't get expired", path);
+ skipDir = true;
+ break;
+ }
+ pathList.add(path);
+ }
+
+ if (skipDir) {
+ continue;
+ }
+ for (int i = 0; i < ruleList.size(); i++) {
+ String path = pathList.get(i);
+ OmLCRule rule = ruleList.get(i);
+ if (currentDir != null && rule.match(currentDir, path)) {
+ if (dirList.isFull()) {
+ // if expiredDirList is full, send delete request for both pending deletion keys and directories
+ handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false);
+ handleAndClearFullList(bucket, dirList, true, scanStateBuilder, false);
+ if (getInjector(2) != null && getInjector(2).getException() != null) {
+ Throwable ex = getInjector(2).getException();
+ getInjector(2).setException(null);
+ throw new RuntimeException(ex);
+ }
+ }
+ dirList.add(currentDirPath, 0, currentDir.getUpdateID());
+ deletedDirSet.add(currentDir.getObjectID());
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ private DirectoryList getSubDirectory(long dirObjID, String prefix, OMMetadataManager metaMgr)
+ throws IOException {
+ DirectoryList subDirList = new DirectoryList();
+
+ // Check all dirTable cache for any sub paths.
+ Table dirTable = metaMgr.getDirectoryTable();
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>>>
+ cacheIter = dirTable.cacheIterator();
+ HashSet<String> deletedDirSet = new HashSet();
+ while (cacheIter.hasNext()) {
+ Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>> entry =
+ cacheIter.next();
+ numDirIterated++;
+ OmDirectoryInfo cacheOmDirInfo = entry.getValue().getCacheValue();
+ if (cacheOmDirInfo == null) {
+ deletedDirSet.add(entry.getKey().getCacheKey());
+ continue;
+ }
+ if (cacheOmDirInfo.getParentObjectID() == dirObjID) {
+ subDirList.addSubDir(entry.getKey().getCacheKey(), cacheOmDirInfo, cacheOmDirInfo.getName());
+ }
+ }
+
+ // Check dirTable entries for any sub paths.
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmDirectoryInfo>>
+ iterator = dirTable.iterator(prefix)) {
+ while (iterator.hasNext()) {
+ numDirIterated++;
+ Table.KeyValue<String, OmDirectoryInfo> entry = iterator.next();
+ OmDirectoryInfo dir = entry.getValue();
+ if (deletedDirSet.contains(entry.getKey())) {
+ continue;
+ }
+ if (dir.getParentObjectID() == dirObjID) {
+ subDirList.addSubDir(entry.getKey(), dir, dir.getName());
+ }
+ }
+ }
+ return subDirList;
+ }
+
+ private void flushAndSaveState(OmBucketInfo bucket, LimitedExpiredObjectList expiredKeyList,
+ LimitedExpiredObjectList expiredDirList, OmLifecycleScanState.Builder scanStateBuilder) {
+ boolean saved = false;
+ if (expiredKeyList != null && !expiredKeyList.isEmpty()) {
+ if (bucket.getBucketLayout() == OBJECT_STORE) {
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), expiredKeyList,
+ false, scanStateBuilder, false);
+ } else {
+ handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, false);
+ }
+ saved = true;
+ }
+ if (expiredDirList != null && !expiredDirList.isEmpty()) {
+ if (bucket.getBucketLayout() != OBJECT_STORE) {
+ handleAndClearFullList(bucket, expiredDirList, true, scanStateBuilder, false);
+ saved = true;
+ }
+ }
+ if (!saved) {
+ sendSaveScanStateRequest(scanStateBuilder, false);
+ }
+ lastStateSaveTime = Time.monotonicNow();
+ lastStateSaveKeyCount = numKeyIterated;
+ }
+
+ private void evaluateBucket(OmBucketInfo bucketInfo,
+ Table<String, OmKeyInfo> keyTable, List<OmLCRule> ruleList, LimitedExpiredObjectList expiredKeyList,
+ OmLifecycleScanState.Builder scanStateBuilder) {
+ String volumeName = bucketInfo.getVolumeName();
+ String bucketName = bucketInfo.getBucketName();
+ String bucketPrefix = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyTblItr =
+ keyTable.iterator(bucketPrefix)) {
+ boolean seekPerformed = false;
+ if (scanStateBuilder != null && scanStateBuilder.getLastScannedKey() != null) {
+ keyTblItr.seek(scanStateBuilder.getLastScannedKey());
+ seekPerformed = true;
+ }
+
+ while (keyTblItr.hasNext()) {
+ if (!shouldRun()) {
+ LOG.info("KeyLifecycleService is suspended or disabled. " +
+ "Stopping LifecycleActionTask for bucket {}.", bucketName);
+ return;
+ }
+ if (shouldSaveState()) {
+ flushAndSaveState(bucketInfo, expiredKeyList, null, scanStateBuilder);
+ }
+ Table.KeyValue<String, OmKeyInfo> keyValue = keyTblItr.next();
+ if (seekPerformed && keyValue.getKey().equals(scanStateBuilder.getLastScannedKey())) {
+ continue;
+ }
+ processKey(bucketInfo, keyValue.getValue(), ruleList, expiredKeyList, scanStateBuilder);
+ numKeyIterated++;
+ lastScannedKey = keyValue.getKey();
+ }
+ } catch (IOException e) {
+ // log failure and continue the process to delete/move files already identified in this run
+ LOG.warn("Failed to iterate through bucket {}/{}", volumeName, bucketName, e);
+ }
+ }
+
+ private void processKey(OmBucketInfo bucketInfo, OmKeyInfo key, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList expiredKeyList, OmLifecycleScanState.Builder scanStateBuilder) {
+ if (bucketInfo.getBucketLayout() == BucketLayout.LEGACY &&
+ key.getKeyName().startsWith(TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)) {
+ return;
+ }
+ for (OmLCRule rule : ruleList) {
+ if (rule.match(key)) {
+ // mark key as expired, check next key
+ if (expiredKeyList.isFull()) {
+ // if expiredKeyList is full, send delete/rename request for expired keys
+ handleAndClearFullList(bucketInfo, expiredKeyList, false, scanStateBuilder, false);
+ if (getInjector(2) != null && getInjector(2).getException() != null) {
+ Throwable ex = getInjector(2).getException();
+ getInjector(2).setException(null);
+ throw new RuntimeException(ex);
+ }
+ }
+ expiredKeyList.add(key.getKeyName(), key.getReplicatedSize(), key.getUpdateID());
+ break;
+ }
+ }
+ }
+
+ /**
+ * Process AbortIncompleteMultipartUpload actions for incomplete multipart uploads.
+ * Iterates through the multipartInfoTable and aborts uploads that match the rule criteria
+ * and have exceeded the configured days after initiation.
+ *
+ * @param bucketInfo the bucket information
+ * @param ruleList list of lifecycle rules with AbortIncompleteMultipartUpload action
+ */
+ private void processMultipartUploads(OmBucketInfo bucketInfo, List<OmLCRule> ruleList) {
+ String volumeName = bucketInfo.getVolumeName();
+ String bucketName = bucketInfo.getBucketName();
+ String bucketPrefix = omMetadataManager.getBucketKeyPrefix(volumeName, bucketName);
+
+ LOG.debug("Processing AbortIncompleteMultipartUpload actions for bucket {}/{}", volumeName, bucketName);
+
+ PartCountLimitedList expiredUploads = new PartCountLimitedList(mpuAbortLimitPerTask);
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmMultipartKeyInfo>> mpuIterator =
+ omMetadataManager.getMultipartInfoTable().iterator(bucketPrefix)) {
+ while (mpuIterator.hasNext()) {
+ if (!shouldRun()) {
+ LOG.info("KeyLifecycleService is suspended or disabled. " +
+ "Stopping multipart upload processing for bucket {}.", bucketName);
+ return;
+ }
+ Table.KeyValue<String, OmMultipartKeyInfo> entry = mpuIterator.next();
+ OmMultipartKeyInfo mpuKeyInfo = entry.getValue();
+ numMultipartUploadIterated++;
+
+ OmMultipartUpload upload;
+ try {
+ upload = OmMultipartUpload.from(entry.getKey());
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Failed to parse multipart upload key {} in bucket {}/{}, skipping",
+ entry.getKey(), volumeName, bucketName, e);
+ continue;
+ }
+
+ upload.setCreationTime(Instant.ofEpochMilli(mpuKeyInfo.getCreationTime()));
+ String keyName = upload.getKeyName();
+
+ String multipartOpenKey;
+ try {
+ multipartOpenKey = OMMultipartUploadUtils.getMultipartOpenKey(
+ volumeName, bucketName, keyName, upload.getUploadId(),
+ omMetadataManager, bucketInfo.getBucketLayout());
+ } catch (OMException e) {
+ LOG.warn("Failed to get multipart open key for {}/{}/{}, skipping",
+ volumeName, bucketName, keyName, e);
+ continue;
+ }
+
+ OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(bucketInfo.getBucketLayout())
+ .get(multipartOpenKey);
+ if (openKeyInfo == null) {
+ LOG.warn("Open key not found for multipart upload {}/{}/{}, skipping",
+ volumeName, bucketName, keyName);
+ continue;
+ }
+
+ for (OmLCRule rule : ruleList) {
+ if (shouldAbortUpload(openKeyInfo, upload, keyName, rule)) {
+ if (expiredUploads.isFull()) {
+ LOG.info("Multipart upload batch reached part count limit {}, aborting current batch " +
+ "({} uploads, {} parts) for bucket {}/{}",
+ mpuAbortLimitPerTask, expiredUploads.size(), expiredUploads.getPartCount(),
+ volumeName, bucketName);
+ abortExpiredMultipartUploadsAndClear(bucketInfo, expiredUploads);
+ }
+
+ // Get part count for this MPU (at least 1 even if no parts uploaded yet)
+ int partCount = Math.max(1, mpuKeyInfo.getPartKeyInfoMap().size());
+ expiredUploads.add(upload, partCount);
+ LOG.debug("Multipart upload {}/{}/{} with uploadId {} ({} parts) will be aborted",
+ volumeName, bucketName, keyName, upload.getUploadId(), partCount);
+ break;
+ }
+ }
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to iterate multipartInfoTable for bucket {}/{}", volumeName, bucketName, e);
+ return;
+ }
+
+ if (!expiredUploads.isEmpty()) {
+ LOG.info("{} expired multipart uploads ({} parts) remaining for bucket {}/{}",
+ expiredUploads.size(), expiredUploads.getPartCount(), volumeName, bucketName);
+ abortExpiredMultipartUploadsAndClear(bucketInfo, expiredUploads);
+ }
+ }
+
+ /**
+ * Check if a multipart upload should be aborted based on the lifecycle rule.
+ *
+ * @param openKeyInfo the open key information with tags
+ * @param upload the multipart upload information
+ * @param keyName the key name of the upload
+ * @param rule the lifecycle rule to evaluate against
+ * @return true if the upload should be aborted, false otherwise
+ */
+ private boolean shouldAbortUpload(OmKeyInfo openKeyInfo, OmMultipartUpload upload,
+ String keyName, OmLCRule rule) {
+
+ if (!rule.getAbortIncompleteMultipartUpload().shouldAbort(
+ upload.getCreationTime().toEpochMilli())) {
+ return false;
+ }
+
+ String effectivePrefix = rule.getEffectivePrefix();
+ if (effectivePrefix != null && !keyName.startsWith(effectivePrefix)) {
+ return false;
+ }
+
+ OmLCFilter filter = rule.getFilter();
+ if (filter != null && !filter.match(openKeyInfo, keyName)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Abort expired multipart uploads by sending an abort request.
+ *
+ * @param bucketInfo the bucket information
+ * @param expiredUploads list of expired multipart uploads to abort
+ */
+ private void abortExpiredMultipartUploads(OmBucketInfo bucketInfo, List<OmMultipartUpload> expiredUploads) {
+ String volumeName = bucketInfo.getVolumeName();
+ String bucketName = bucketInfo.getBucketName();
+
+ List<OzoneManagerProtocolProtos.ExpiredMultipartUploadInfo> expiredMPUInfoList = expiredUploads.stream()
+ .map(upload -> OzoneManagerProtocolProtos.ExpiredMultipartUploadInfo.newBuilder()
+ .setName(upload.getDbKey())
+ .build())
+ .collect(Collectors.toList());
+
+ OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket expiredMPUBucket =
+ OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .addAllMultipartUploads(expiredMPUInfoList)
+ .build();
+
+ OzoneManagerProtocolProtos.MultipartUploadsExpiredAbortRequest abortRequest =
+ OzoneManagerProtocolProtos.MultipartUploadsExpiredAbortRequest.newBuilder()
+ .addExpiredMultipartUploadsPerBucket(expiredMPUBucket)
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.AbortExpiredMultiPartUploads)
+ .setMultipartUploadsExpiredAbortRequest(abortRequest)
+ .setVersion(ClientVersion.CURRENT_VERSION)
+ .setClientId(clientId.toString())
+ .build();
+
+ try {
+ long startTime = System.nanoTime();
+ OzoneManagerProtocolProtos.OMResponse response = OzoneManagerRatisUtils.submitRequest(
+ getOzoneManager(), omRequest, clientId, callId.getAndIncrement());
+ long endTime = System.nanoTime();
+
+ if (response != null) {
+ if (response.getSuccess()) {
+ numMultipartUploadAborted += expiredUploads.size();
+
+ LOG.info("Successfully aborted {} multipart uploads for bucket {}/{} in {} ns",
+ expiredUploads.size(), volumeName, bucketName, endTime - startTime);
+ } else {
+ LOG.error("Failed to abort multipart uploads for bucket {}/{}: {}",
+ volumeName, bucketName, response.getMessage());
+ }
+ } else {
+ LOG.error("Received null response when aborting multipart uploads for bucket {}/{}",
+ volumeName, bucketName);
+ }
+ } catch (ServiceException e) {
+ LOG.error("Failed to submit abort multipart uploads request for bucket {}/{}",
+ volumeName, bucketName, e);
+ }
+ }
+
+ private void abortExpiredMultipartUploadsAndClear(OmBucketInfo bucketInfo,
+ PartCountLimitedList expiredUploads) {
+ if (expiredUploads.isEmpty()) {
+ return;
+ }
+
+ abortExpiredMultipartUploads(bucketInfo, expiredUploads.getUploads());
+ expiredUploads.clear();
+ }
+
+ /**
+ * If the prefix is /dir1/dir2, but dir1 doesn't exist, then it will return an exception.
+ * If the prefix is /dir1/dir2, but dir2 doesn't exist, then it will return a list with dir1 only.
+ * If the prefix is /dir1/dir2, although dir1 exists, but get(dir1) failed with IOException,
+ * then it will return an exception too.
+ */
+ private DirectoryList getDirList(long volumeID, OmBucketInfo bucket, String prefix, String bucketKey)
+ throws IOException {
+ // find KeyInfo of each directory for the prefix
+ java.nio.file.Path keyPath = Paths.get(prefix);
+ Iterator<java.nio.file.Path> elements = keyPath.iterator();
+ long lastKnownParentId = bucket.getObjectID();
+ DirectoryList directoryList = new DirectoryList();
+ StringBuffer currentDirPath = new StringBuffer();
+ while (elements.hasNext()) {
+ String dirName = elements.next().toString();
+ String dbDirName = omMetadataManager.getOzonePathKey(
+ volumeID, bucket.getObjectID(), lastKnownParentId, dirName);
+ try {
+ OmDirectoryInfo omDirInfo = omMetadataManager.getDirectoryTable().get(dbDirName);
+ // It's OK there is no directory for the last part of the prefix, which is probably not a directory
+ if (omDirInfo == null) {
+ if (elements.hasNext()) {
+ throw new OMException("Directory " + dbDirName + " does not exist for bucket " + bucketKey,
+ OMException.ResultCodes.DIRECTORY_NOT_FOUND);
+ }
+ directoryList.setNotAllResolvedPrefix();
+ } else {
+ if (currentDirPath.length() == 0) {
+ currentDirPath.append(dirName);
+ } else {
+ currentDirPath.append('/').append(dirName);
+ }
+ directoryList.addSubDir(dbDirName, omDirInfo, currentDirPath.toString());
+ lastKnownParentId = omDirInfo.getObjectID();
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to get directory {} for bucket {}", dbDirName, bucketKey, e);
+ throw new IOException("Failed to get directory " + dbDirName + " for bucket " + bucketKey);
+ }
+ }
+ return directoryList;
+ }
+
+ private void onFailure(String bucketName) {
+ inFlight.remove(bucketName);
+ metrics.incrNumFailureTask();
+ metrics.incNumKeyIterated(numKeyIterated);
+ metrics.incNumDirIterated(numDirIterated);
+ metrics.incNumMultipartUploadIterated(numMultipartUploadIterated);
+ long timeSpent = Time.monotonicNow() - taskStartTime;
+ LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " +
+ "deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " +
+ "aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated,
+ numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted,
+ numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted);
+ }
+
+ private void onSuccess(String bucketName) {
+ inFlight.remove(bucketName);
+ metrics.incrNumSuccessTask();
+ long timeSpent = Time.monotonicNow() - taskStartTime;
+ metrics.incTaskLatencyMs(timeSpent);
+ metrics.incNumKeyIterated(numKeyIterated);
+ metrics.incNumDirIterated(numDirIterated);
+ metrics.incNumMultipartUploadIterated(numMultipartUploadIterated);
+ metrics.incNumMultipartUploadAborted(numMultipartUploadAborted);
+ LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " +
+ "deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " +
+ "aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated,
+ numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted,
+ numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted);
+ }
+
+ private void handleAndClearFullList(OmBucketInfo bucket, LimitedExpiredObjectList keysList,
+ boolean dir, OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
+ if (moveToTrashEnabled.get() && bucket.getBucketLayout() != OBJECT_STORE && ozoneTrash != null) {
+ moveToTrash(bucket, keysList, dir);
+ sendSaveScanStateRequest(scanStateBuilder, scanFinished);
+ } else {
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), keysList, dir,
+ scanStateBuilder, scanFinished);
+ }
+ }
+
+ private void sendSaveScanStateRequest(OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
+ if (scanStateBuilder != null) {
+ if (lastScannedDir != null) {
+ scanStateBuilder.setLastScannedDir(lastScannedDir);
+ }
+ if (lastScannedDirKey != null) {
+ scanStateBuilder.setLastScannedDirKey(lastScannedDirKey);
+ }
+ if (lastScannedKey != null) {
+ scanStateBuilder.setLastScannedKey(lastScannedKey);
+ }
+ if (scanFinished) {
+ scanStateBuilder.setScanEndTime(System.currentTimeMillis());
+ }
+ OmLifecycleScanState state = scanStateBuilder.build();
+
+ SaveLifecycleScanStateRequest saveRequest = SaveLifecycleScanStateRequest.newBuilder()
+ .setState(state.getProtobuf())
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState)
+ .setVersion(ClientVersion.CURRENT_VERSION)
+ .setClientId(clientId.toString())
+ .setSaveLifecycleScanStateRequest(saveRequest)
+ .build();
+
+ LOG.debug("Save scan state {}", state);
+ try {
+ OzoneManagerRatisUtils.submitRequest(getOzoneManager(), omRequest, clientId, callId.getAndIncrement());
+ } catch (ServiceException e) {
+ LOG.error("Failed to submit SaveLifecycleScanState request", e);
+ }
+ }
+ }
+
+ private void sendDeleteKeysRequestAndClearList(String volume, String bucket, LimitedExpiredObjectList keysList,
+ boolean dir, OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
+ try {
+ if (getInjector(1) != null) {
+ try {
+ getInjector(1).pause();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ int batchSize = keyDeleteBatchSize;
+ int startIndex = 0;
+ for (int i = 0; i < keysList.size();) {
+ DeleteKeyArgs.Builder builder =
+ DeleteKeyArgs.newBuilder().setBucketName(bucket).setVolumeName(volume);
+ int endIndex = startIndex + (batchSize < (keysList.size() - startIndex) ?
+ batchSize : keysList.size() - startIndex);
+ int keyCount = endIndex - startIndex;
+ builder.addAllKeys(keysList.nameSubList(startIndex, endIndex));
+ builder.addAllUpdateIDs(keysList.updateIDSubList(startIndex, endIndex));
+
+ DeleteKeyArgs deleteKeyArgs = builder.build();
+ DeleteKeysRequest.Builder requestBuilder = DeleteKeysRequest.newBuilder()
+ .setDeleteKeys(deleteKeyArgs)
+ .setSourceType(RequestSource.LIFECYCLE);
+
+ if (scanStateBuilder != null) {
+ if (lastScannedKey != null) {
+ scanStateBuilder.setLastScannedKey(lastScannedKey);
+ }
+ if (lastScannedDir != null) {
+ scanStateBuilder.setLastScannedDir(lastScannedDir);
+ }
+ if (lastScannedDirKey != null) {
+ scanStateBuilder.setLastScannedDirKey(lastScannedDirKey);
+ }
+ if (scanFinished) {
+ scanStateBuilder.setScanEndTime(System.currentTimeMillis());
+ }
+ OmLifecycleScanState state = scanStateBuilder.build();
+ requestBuilder.setScanState(state.getProtobuf());
+ LOG.debug("Save scan state: {}", state);
+ }
+
+ DeleteKeysRequest deleteKeysRequest = requestBuilder.build();
+ LOG.debug("request size {} for {} keys", deleteKeysRequest.getSerializedSize(), keyCount);
+
+ if (deleteKeysRequest.getSerializedSize() < ratisByteLimit) {
+ // send request out
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKeys)
+ .setVersion(ClientVersion.CURRENT_VERSION)
+ .setClientId(clientId.toString())
+ .setDeleteKeysRequest(deleteKeysRequest)
+ .build();
+ long startTime = System.nanoTime();
+ final OzoneManagerProtocolProtos.OMResponse response = OzoneManagerRatisUtils.submitRequest(
+ getOzoneManager(), omRequest, clientId, callId.getAndIncrement());
+ long endTime = System.nanoTime();
+ LOG.debug("DeleteKeys request with {} keys cost {} ns", keyCount, endTime - startTime);
+ long deletedCount = keyCount;
+ long deletedSize = keysList.replicatedSizeSubList(startIndex, endIndex)
+ .stream().mapToLong(Long::longValue).sum();
+ if (response != null) {
+ if (!response.getSuccess()) {
+ // log the failure and continue the iterating
+ LOG.error("DeleteKeys request " + response.getStatus() + " failed for volume: {}, bucket: {}",
+ volume, bucket);
+ if (response.getDeleteKeysResponse().hasUnDeletedKeys()) {
+ DeleteKeyArgs unDeletedKeys = response.getDeleteKeysResponse().getUnDeletedKeys();
+ for (String key : unDeletedKeys.getKeysList()) {
+ Long size = keysList.getReplicatedSize(key);
+ if (size == null) {
+ LOG.error("Undeleted key {}/{}/{} doesn't in keyLists", volume, bucket, key);
+ continue;
+ }
+ deletedCount -= 1;
+ deletedSize -= size;
+ }
+ }
+ for (DeleteKeyError e : response.getDeleteKeysResponse().getErrorsList()) {
+ Long size = keysList.getReplicatedSize(e.getKey());
+ if (size == null) {
+ LOG.error("Deleted error key {}/{}/{} doesn't in keyLists", volume, bucket, e.getKey());
+ continue;
+ }
+ deletedCount -= 1;
+ deletedSize -= size;
+ }
+ } else {
+ LOG.debug("DeleteKeys request of total {} keys, {} not deleted", keyCount,
+ response.getDeleteKeysResponse().getErrorsCount());
+ }
+ }
+ if (dir) {
+ numDirDeleted += deletedCount;
+ metrics.incrNumDirDeleted(deletedCount);
+ } else {
+ numKeyDeleted += deletedCount;
+ sizeKeyDeleted += deletedSize;
+ metrics.incrNumKeyDeleted(deletedCount);
+ metrics.incrSizeKeyDeleted(deletedSize);
+ }
+ i += keyCount;
+ startIndex += keyCount;
+ } else {
+ batchSize /= 2;
+ }
+ }
+ } catch (ServiceException e) {
+ LOG.error("Failed to send DeleteKeysRequest", e);
+ } finally {
+ keysList.clear();
+ }
+ }
+
+ private void moveToTrash(OmBucketInfo bucket, LimitedExpiredObjectList keysList, boolean isDir) {
+ if (keysList.isEmpty()) {
+ return;
+ }
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ String trashRoot = TRASH_PREFIX + OM_KEY_PREFIX + bucket.getOwner();
+ Path trashCurrent = new Path(trashRoot, CURRENT);
+ try {
+ checkAndCreateTrashDirIfNeeded(bucket, trashCurrent);
+ } catch (IOException e) {
+ keysList.clear();
+ return;
+ }
+
+ for (int i = 0; i < keysList.size(); i++) {
+ String keyName = keysList.getName(i);
+ Path keyPath = new Path(OzoneConsts.OZONE_URI_DELIMITER + keyName);
+ Path baseKeyTrashPath = Path.mergePaths(trashCurrent, keyPath.getParent());
+ try {
+ checkAndCreateTrashDirIfNeeded(bucket, baseKeyTrashPath);
+ } catch (IOException e) {
+ LOG.error("Failed to check and create Trash dir {} for bucket {}/{}", baseKeyTrashPath,
+ volumeName, bucketName, e);
+ continue;
+ }
+ String targetKeyName = trashCurrent + OM_KEY_PREFIX + keyName;
+ KeyArgs keyArgs = KeyArgs.newBuilder().setKeyName(keyName)
+ .setVolumeName(volumeName).setBucketName(bucketName).build();
+
+ /**
+ * Trash examples:
+ * /s3v/test/readme -> /s3v/test/.Trash/hadoop/Current/readme
+ * /s3v/test/dir1/readme -> /s3v/test/.Trash/hadoop/Current/dir1/readme
+ */
+ RenameKeyRequest renameKeyRequest = RenameKeyRequest.newBuilder()
+ .setKeyArgs(keyArgs)
+ .setToKeyName(targetKeyName)
+ .setUpdateID(keysList.getUpdateID(i))
+ .build();
+
+ // send request out
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
+ .setVersion(ClientVersion.CURRENT_VERSION)
+ .setClientId(clientId.toString())
+ .setRenameKeyRequest(renameKeyRequest)
+ .build();
+ try {
+ // perform preExecute as ratis submit do no perform preExecute
+ OMClientRequest omClientRequest = OzoneManagerRatisUtils.createClientRequest(omRequest, ozoneManager);
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(bucket.getOwner());
+ OzoneManagerProtocolProtos.OMResponse omResponse =
+ ugi.doAs(new PrivilegedExceptionAction<OzoneManagerProtocolProtos.OMResponse>() {
+ @Override
+ public OzoneManagerProtocolProtos.OMResponse run() throws Exception {
+ OMRequest request = omClientRequest.preExecute(ozoneManager);
+ return OzoneManagerRatisUtils.submitRequest(getOzoneManager(),
+ request, clientId, callId.getAndIncrement());
+ }
+ });
+ if (omResponse != null) {
+ if (!omResponse.getSuccess()) {
+ // log the failure and continue the iterating
+ LOG.error("RenameKey request failed with source key: {}, dest key: {}", keyName, targetKeyName);
+ continue;
+ }
+ }
+ LOG.info("RenameKey request succeed with source key: {}, dest key: {}", keyName, targetKeyName);
+
+ if (isDir) {
+ numDirRenamed += 1;
+ metrics.incrNumDirRenamed(1);
+ } else {
+ numKeyRenamed += 1;
+ sizeKeyRenamed += keysList.getReplicatedSize(i);
+ metrics.incrNumKeyRenamed(1);
+ metrics.incrSizeKeyRenamed(keysList.getReplicatedSize(i));
+ }
+ } catch (IOException | InterruptedException e) {
+ LOG.error("Failed to send RenameKeysRequest", e);
+ }
+ }
+ keysList.clear();
+ }
+
+ private void checkAndCreateTrashDirIfNeeded(OmBucketInfo bucket, Path dirPath) throws IOException {
+ OmKeyArgs key = new OmKeyArgs.Builder().setVolumeName(bucket.getVolumeName())
+ .setBucketName(bucket.getBucketName()).setKeyName(dirPath.toString())
+ .setOwnerName(bucket.getOwner()).build();
+ try {
+ ozoneManager.getFileStatus(key);
+ } catch (IOException e) {
+ if (e instanceof OMException &&
+ (((OMException) e).getResult() == OMException.ResultCodes.FILE_NOT_FOUND ||
+ ((OMException) e).getResult() == OMException.ResultCodes.DIRECTORY_NOT_FOUND)) {
+ // create the trash/Current directory for user
+ KeyArgs keyArgs = KeyArgs.newBuilder().setVolumeName(bucket.getVolumeName())
+ .setBucketName(bucket.getBucketName()).setKeyName(dirPath.toString())
+ .setOwnerName(bucket.getOwner()).setRecursive(true).build();
+ OMRequest omRequest = OMRequest.newBuilder().setCreateDirectoryRequest(
+ CreateDirectoryRequest.newBuilder().setKeyArgs(keyArgs))
+ .setCmdType(OzoneManagerProtocolProtos.Type.CreateDirectory)
+ .setVersion(ClientVersion.CURRENT_VERSION)
+ .setClientId(clientId.toString())
+ .build();
+ try {
+ // perform preExecute as ratis submit do no perform preExecute
+ final OMClientRequest omClientRequest = OzoneManagerRatisUtils.createClientRequest(omRequest, ozoneManager);
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(bucket.getOwner());
+ OzoneManagerProtocolProtos.OMResponse omResponse =
+ ugi.doAs(new PrivilegedExceptionAction<OzoneManagerProtocolProtos.OMResponse>() {
+ @Override
+ public OzoneManagerProtocolProtos.OMResponse run() throws Exception {
+ OMRequest request = omClientRequest.preExecute(ozoneManager);
+ return OzoneManagerRatisUtils.submitRequest(getOzoneManager(),
+ request, clientId, callId.getAndIncrement());
+ }
+ });
+
+ if (omResponse != null) {
+ if (!omResponse.getSuccess()) {
+ LOG.error("CreateDirectory request failed with {}, path: {}",
+ omResponse.getMessage(), dirPath);
+ throw new IOException("Failed to create trash directory " + dirPath);
+ }
+ }
+ LOG.info("Created directory {}/{}/{}", bucket.getVolumeName(), bucket.getBucketName(), dirPath);
- } catch (InterruptedException | IOException e1) {
++ } catch (IOException | InterruptedException e1) {
+ LOG.error("Failed to send CreateDirectoryRequest for {}", dirPath, e1);
+ throw new IOException("Failed to send CreateDirectoryRequest request for " + dirPath);
+ }
+ } else {
+ LOG.error("Failed to get trash current directory {} status", dirPath, e);
+ throw e;
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public static FaultInjector getInjector(int index) {
+ return injectors != null ? injectors.get(index) : null;
+ }
+
+ @VisibleForTesting
+ public static void setInjectors(List<FaultInjector> instance) {
+ injectors = instance;
+ }
+
+ @VisibleForTesting
+ public static Logger getLog() {
+ return LOG;
+ }
+
+ @VisibleForTesting
+ public void setListMaxSize(int size) {
+ this.listMaxSize = size;
+ }
+
+ @VisibleForTesting
+ public void setMpuAbortLimitPerTask(int limit) {
+ this.mpuAbortLimitPerTask = limit;
+ }
+
+ @VisibleForTesting
+ public void setOzoneTrash(OzoneTrash ozoneTrash) {
+ this.ozoneTrash = ozoneTrash;
+ }
+
+ @VisibleForTesting
+ public void setMoveToTrashEnabled(boolean enabled) {
+ this.moveToTrashEnabled.set(enabled);
+ }
+
+ /**
+ * An in-memory list with limited size to hold expired object infos, including object name and current update ID.
+ */
+ public static class LimitedExpiredObjectList {
+ private final LimitedSizeList<String> objectNames;
+ private final List<Long> objectReplicatedSize;
+ private final List<Long> objectUpdateIDs;
+
+ public LimitedExpiredObjectList(int maxListSize) {
+ this.objectNames = new LimitedSizeList<>(maxListSize);
+ this.objectReplicatedSize = new ArrayList<>();
+ this.objectUpdateIDs = new ArrayList<>();
+ }
+
+ public void add(String name, long size, long updateID) {
+ objectNames.add(name);
+ objectReplicatedSize.add(size);
+ objectUpdateIDs.add(updateID);
+ }
+
+ public void addAll(LimitedExpiredObjectList other) {
+ objectNames.addAll(other.objectNames);
+ objectReplicatedSize.addAll(other.objectReplicatedSize);
+ objectUpdateIDs.addAll(other.objectUpdateIDs);
+ }
+
+ public int size() {
+ return objectNames.size();
+ }
+
+ public List<String> nameSubList(int fromIndex, int toIndex) {
+ return objectNames.subList(fromIndex, toIndex);
+ }
+
+ public List<Long> updateIDSubList(int fromIndex, int toIndex) {
+ return objectUpdateIDs.subList(fromIndex, toIndex);
+ }
+
+ public List<Long> replicatedSizeSubList(int fromIndex, int toIndex) {
+ return objectReplicatedSize.subList(fromIndex, toIndex);
+ }
+
+ public Long getReplicatedSize(String keyName) {
+ for (int index = 0; index < objectNames.size(); index++) {
+ if (objectNames.get(index).equals(keyName)) {
+ return objectReplicatedSize.get(index);
+ }
+ }
+ return null;
+ }
+
+ public void clear() {
+ objectNames.clear();
+ objectUpdateIDs.clear();
+ objectReplicatedSize.clear();
+ }
+
+ public boolean isEmpty() {
+ return objectNames.isEmpty();
+ }
+
+ public boolean isFull() {
+ return objectNames.isFull();
+ }
+
+ public String getName(int index) {
+ return objectNames.get(index);
+ }
+
+ public long getUpdateID(int index) {
+ return objectUpdateIDs.get(index);
+ }
+
+ public long getReplicatedSize(int index) {
+ return objectReplicatedSize.get(index);
+ }
+ }
+
+ /**
+ * An in-memory list with a maximum size. This class is not thread safe.
+ */
+ public static class LimitedSizeList<T> {
+ private final List<T> internalList;
+ private final int maxSize;
+
+ public LimitedSizeList(int maxSize) {
+ this.maxSize = maxSize;
+ this.internalList = new ArrayList<>();
+ }
+
+ /**
+ * Add an element to the list. It blindly adds the element without check whether the list is full or not.
+ * Caller must check the size of the list through isFull() before calling this method.
+ */
+ public void add(T element) {
+ internalList.add(element);
+ }
+
+ public void addAll(LimitedSizeList<T> other) {
+ internalList.addAll(other.internalList);
+ }
+
+ public T get(int index) {
+ return internalList.get(index);
+ }
+
+ public int size() {
+ return internalList.size();
+ }
+
+ public List<T> subList(int fromIndex, int toIndex) {
+ return internalList.subList(fromIndex, toIndex);
+ }
+
+ public boolean isEmpty() {
+ return internalList.isEmpty();
+ }
+
+ public boolean isFull() {
+ boolean full = internalList.size() >= maxSize;
+ if (full) {
+ LOG.debug("LimitedSizeList has reached maximum size {}", maxSize);
+ }
+ return full;
+ }
+
+ public void clear() {
+ internalList.clear();
+ }
+ }
+
+ /**
+ * A list that tracks the total part count of multipart uploads.
+ * The list is considered "full" when the total part count reaches the limit.
+ * This is used because some MPUs might have 1 part while others might have 10,000 parts.
+ */
+ public static class PartCountLimitedList {
+ private final List<OmMultipartUpload> uploads;
+ private final int maxPartCount;
+ private int currentPartCount;
+
+ public PartCountLimitedList(int maxPartCount) {
+ this.maxPartCount = maxPartCount;
+ this.uploads = new ArrayList<>();
+ this.currentPartCount = 0;
+ }
+
+ /**
+ * Add a multipart upload with its part count.
+ * Caller should check isFull() before calling this method.
+ */
+ public void add(OmMultipartUpload upload, int partCount) {
+ uploads.add(upload);
+ currentPartCount += partCount;
+ }
+
+ public List<OmMultipartUpload> getUploads() {
+ return uploads;
+ }
+
+ public int size() {
+ return uploads.size();
+ }
+
+ public int getPartCount() {
+ return currentPartCount;
+ }
+
+ public boolean isEmpty() {
+ return uploads.isEmpty();
+ }
+
+ public boolean isFull() {
+ boolean full = currentPartCount >= maxPartCount;
+ if (full) {
+ LOG.debug("PartCountLimitedList has reached maximum part count {}", maxPartCount);
+ }
+ return full;
+ }
+
+ public void clear() {
+ uploads.clear();
+ currentPartCount = 0;
+ }
+ }
+
+ /**
+ * An in-memory class to hold the information required in directory recursive evaluation.
+ */
+ public static class PendingEvaluateDirectory {
+ private final OmDirectoryInfo directoryInfo;
+ private String dirTableKey;
+ private String dirPath;
+ private DirectoryList directoryList;
+ private boolean firstEvaluate;
+
+ public PendingEvaluateDirectory(OmDirectoryInfo dir, String dirTableKey, String dirPath, DirectoryList summary) {
+ this.directoryInfo = dir;
+ this.dirTableKey = dirTableKey;
+ this.dirPath = dirPath;
+ this.directoryList = summary;
+ this.firstEvaluate = true;
+ }
+
+ public String getDirTableKey() {
+ return dirTableKey;
+ }
+
+ public void setDirTableKey(String tableKey) {
+ dirTableKey = tableKey;
+ }
+
+ public String getDirPath() {
+ return dirPath;
+ }
+
+ public OmDirectoryInfo getDirectoryInfo() {
+ return directoryInfo;
+ }
+
+ public DirectoryList getSubDirSummary() {
+ return directoryList;
+ }
+
+ public void setDirectoryList(DirectoryList summary) {
+ directoryList = summary;
+ }
+
+ public boolean isFirstEvaluate() {
+ return firstEvaluate;
+ }
+
+ public void setFirstEvaluate(boolean firstEvaluate) {
+ this.firstEvaluate = firstEvaluate;
+ }
+ }
+
+ /**
+ * An in-memory class to hold a directory list.
+ */
+ public static class DirectoryList {
+ private final List<OmDirectoryInfo> subDirList;
+ private final List<String> subDirKeyList;
+ private final List<String> subDirKeyPathList;
+ private int subDirCount;
+ private boolean allResolvedPrefix = true;
+
+ public DirectoryList() {
+ this.subDirList = new ArrayList<>();
+ this.subDirKeyList = new ArrayList<>();
+ this.subDirKeyPathList = new ArrayList<>();
+ this.subDirCount = 0;
+ }
+
+ public int getSubDirCount() {
+ return subDirCount;
+ }
+
+ public List<OmDirectoryInfo> getSubDirList() {
+ return subDirList;
+ }
+
+ public List<String> getSubDirKeyList() {
+ return subDirKeyList;
+ }
+
+ public void addSubDir(String key, OmDirectoryInfo dir, String dirPath) {
+ subDirKeyList.add(key);
+ subDirList.add(dir);
+ subDirKeyPathList.add(dirPath);
+ subDirCount++;
+ }
+
+ public OmDirectoryInfo getLastSubDir() {
+ return subDirCount > 0 ? subDirList.get(subDirCount - 1) : null;
+ }
+
+ public String getLastSubDirKey() {
+ return subDirCount > 0 ? subDirKeyList.get(subDirCount - 1) : null;
+ }
+
+ public String getLastSubDirPath() {
+ return subDirCount > 0 ? subDirKeyPathList.get(subDirCount - 1) : null;
+ }
+
+ public boolean isAllResolvedPrefix() {
+ return allResolvedPrefix;
+ }
+
+ public void setNotAllResolvedPrefix() {
+ this.allResolvedPrefix = false;
+ }
+
+ public boolean isEmpty() {
+ return subDirCount == 0;
+ }
+
+ @Override
+ public String toString() {
+ return "DirectoryList { " +
+ "subDirList = " + subDirList +
+ ", subDirKeyList = " + subDirKeyList +
+ ", subDirKeyPathList = " + subDirKeyPathList +
+ ", subDirCount = " + subDirCount +
+ ", allResolvedPrefix = " + allResolvedPrefix +
+ '}';
+ }
+ }
+
+ /**
+ * An in-memory class to hold a rule list, together with the resolved prefix's directory list.
+ */
+ public static class RuleListWithDirectoryList {
+ private DirectoryList dirList;
+ private final List<OmLCRule> ruleList;
+ private String consolidatedPrefix;
+
+ public RuleListWithDirectoryList() {
+ this.ruleList = new ArrayList<>();
+ this.dirList = new DirectoryList();
+ }
+
+ public RuleListWithDirectoryList(List<OmLCRule> ruleList, DirectoryList dirList, String prefix) {
+ this.ruleList = ruleList;
+ this.dirList = dirList;
+ this.consolidatedPrefix = prefix;
+ }
+
+ public List<OmLCRule> getRuleList() {
+ return ruleList;
+ }
+
+ public DirectoryList getDirList() {
+ return dirList;
+ }
+
+ public void addRule(OmLCRule rule) {
+ this.ruleList.add(rule);
+ }
+
+ public void setDirList(DirectoryList dirList) {
+ this.dirList = dirList;
+ }
+
+ public String getConsolidatedPrefix() {
+ return consolidatedPrefix;
+ }
+
+ public void setConsolidatedPrefix(String consolidatedPrefix) {
+ this.consolidatedPrefix = consolidatedPrefix;
+ }
+
+ public boolean isEmpty() {
+ return dirList.isEmpty() && ruleList.isEmpty() && consolidatedPrefix == null;
+ }
+
+ @Override
+ public String toString() {
+ return "RuleListWithDirectoryList { " +
+ "dirList = " + dirList +
+ ", ruleList = " + ruleList +
+ ", consolidatedPrefix = '" + consolidatedPrefix + '\'' +
+ '}';
+ }
+ }
+
+ /**
+ * Orders of RuleListWithDirectoryList.
+ */
+ public static class RuleListWithDirectoryListOrder implements Comparator<RuleListWithDirectoryList> {
+
+ public static final Comparator<RuleListWithDirectoryList> INSTANCE =
+ new RuleListWithDirectoryListOrder();
+
+ @Override
+ public int compare(RuleListWithDirectoryList o1, RuleListWithDirectoryList o2) {
+ List<OmDirectoryInfo> dirList1 = o1.getDirList().getSubDirList();
+ List<OmDirectoryInfo> dirList2 = o2.getDirList().getSubDirList();
+
+ // find their shared parent directory
+ OmDirectoryInfo parent = null;
+ for (int i = dirList1.size() - 1; i >= 0; i--) {
+ long objID = dirList1.get(i).getObjectID();
+ for (int j = dirList2.size() - 1; j >= 0; j--) {
+ if (dirList2.get(j).getObjectID() == objID) {
+ parent = dirList2.get(j);
+ break;
+ }
+ }
+ if (parent != null) {
+ break;
+ }
+ }
+ if (parent == null) {
+ // e.g, dir1 and dir2, dir2 should ahead of dir2, given the depth-first and dir's RocksDB key order
+ return dirList2.get(0).getName().compareTo(dirList1.get(0).getName());
+ } else {
+ long parentID = parent.getObjectID();
+ OmDirectoryInfo dir1 = dirList1.stream().filter(dir -> dir.getParentObjectID() == parentID)
+ .collect(Collectors.toList()).get(0);
+ OmDirectoryInfo dir2 = dirList2.stream().filter(dir -> dir.getParentObjectID() == parentID)
+ .collect(Collectors.toList()).get(0);
+ return dir2.getName().compareTo(dir1.getName());
+ }
+ }
+ }
+
+ /**
+ * An in-memory stack with a maximum size. This class is not thread safe.
+ */
+ public static class LimitedSizeStack {
+ private final Deque<PendingEvaluateDirectory> stack;
+ private final long maxSize;
+
+ public LimitedSizeStack(long maxSize) {
+ this.maxSize = maxSize;
+ this.stack = new ArrayDeque<>();
+ }
+
+ public boolean isEmpty() {
+ return stack.isEmpty();
+ }
+
+ public void push(PendingEvaluateDirectory e) throws CapacityFullException {
+ if (stack.size() >= maxSize) {
+ throw new CapacityFullException("LimitedSizeStack has reached maximum size " + maxSize);
+ }
+ stack.push(e);
+ }
+
+ public PendingEvaluateDirectory pop() {
+ return stack.pop();
+ }
+ }
+
+ /**
+ * An exception which indicates the collection is full.
+ */
+ public static class CapacityFullException extends Exception {
+ public CapacityFullException(String message) {
+ super(message);
+ }
+ }
+
+ public static void setTest(boolean test) {
+ KeyLifecycleService.test = test;
+ }
+
+ public static void reSetConsolidatedRuleList() {
+ consolidatedRuleList = null;
+ }
+
+ public static List<RuleListWithDirectoryList> getConsolidatedRuleList() {
+ return consolidatedRuleList;
+ }
+}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 4bb0ad43e9c,760dd90e9ce..57e52ab2d91
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@@ -402,19 -402,11 +406,24 @@@ public class OzoneManagerRequestHandle
getObjectTagging(request.getGetObjectTaggingRequest());
responseBuilder.setGetObjectTaggingResponse(getObjectTaggingResponse);
break;
+ case GetLifecycleConfiguration:
+ GetLifecycleConfigurationResponse getLifecycleConfigurationResponse =
+ infoLifecycleConfiguration(
+ request.getGetLifecycleConfigurationRequest());
+ responseBuilder.setGetLifecycleConfigurationResponse(
+ getLifecycleConfigurationResponse);
+ break;
+ case GetLifecycleServiceStatus:
+ GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse =
+ impl.getLifecycleServiceStatus();
+ responseBuilder.setGetLifecycleServiceStatusResponse(
+ getLifecycleServiceStatusResponse);
+ break;
+ case GetBucketTagging:
+ GetBucketTaggingResponse getBucketTaggingResponse =
+ getBucketTagging(request.getGetBucketTaggingRequest());
+ responseBuilder.setGetBucketTaggingResponse(getBucketTaggingResponse);
+ break;
default:
responseBuilder.setSuccess(false);
responseBuilder.setMessage("Unrecognized Command Type: " + cmdType);
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
index 08bc2c2e1bd,0ab5a1004ba..e46f4d53f9f
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
@@@ -31,9 -31,7 +31,10 @@@ public enum S3GAction implements AuditA
PUT_ACL,
LIST_MULTIPART_UPLOAD,
MULTI_DELETE,
+ GET_BUCKET_LOCATION,
+ GET_BUCKET_LIFECYCLE,
+ PUT_BUCKET_LIFECYCLE,
+ DELETE_BUCKET_LIFECYCLE,
//RootEndpoint
LIST_S3_BUCKETS,
diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
index c91fb91421f,41a19231a82..1e0f243a49b
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
@@@ -58,24 -47,10 +58,25 @@@ public class BucketCrudHandler extends
private boolean shouldHandle() {
return queryParams().get(QueryParams.ACL) == null
&& queryParams().get(QueryParams.UPLOADS) == null
- && queryParams().get(QueryParams.DELETE) == null;
+ && queryParams().get(QueryParams.DELETE) == null
+ && queryParams().get(QueryParams.TAGGING) == null;
}
+ /**
+ * Handle GET /{bucket} for bucket LIFECYCLE configuration.
+ */
+ @Override
+ Response handleGetRequest(S3RequestContext context, String bucketName)
+ throws IOException, OS3Exception {
+ final String lifecycleMarker = queryParams().get(QueryParams.LIFECYCLE);
+
+ if (lifecycleMarker != null) {
+ context.setAction(S3GAction.GET_BUCKET_LIFECYCLE);
+ return getBucketLifecycleConfiguration(context, bucketName);
+ }
+ return null;
+ }
+
/**
* Handle PUT /{bucket} for bucket creation.
*/
diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index 35add7c99f8,8cc421c5c6e..cd15256e452
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@@ -877,21 -879,19 +880,36 @@@ public class ClientProtocolStub impleme
getBucket(volumeName, bucketName).deleteObjectTagging(keyName);
}
+ @Override
+ public OzoneLifecycleConfiguration getLifecycleConfiguration(String volumeName, String bucketName)
+ throws IOException {
+ return getBucket(volumeName, bucketName).getLifecycleConfiguration();
+ }
+
+ @Override
+ public void setLifecycleConfiguration(OmLifecycleConfiguration lifecycleConfiguration) throws IOException {
+ getBucket(lifecycleConfiguration.getVolume(), lifecycleConfiguration.getBucket())
+ .setLifecycleConfiguration(lifecycleConfiguration);
+ }
+
+ @Override
+ public void deleteLifecycleConfiguration(String volumeName, String bucketName) throws IOException {
+
+ }
+
+ @Override
+ public Map<String, String> getBucketTagging(String volumeName, String bucketName) throws IOException {
+ return getBucket(volumeName, bucketName).getBucketTagging();
+ }
+
+ @Override
+ public void putBucketTagging(String volumeName, String bucketName, Map<String, String> tags)
+ throws IOException {
+ getBucket(volumeName, bucketName).putBucketTagging(tags);
+ }
+
+ @Override
+ public void deleteBucketTagging(String volumeName, String bucketName) throws IOException {
+ getBucket(volumeName, bucketName).deleteBucketTagging();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org