You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2023/02/01 03:13:52 UTC
[ozone] 01/01: Merge remote-tracking branch 'asf/master' into HDDS-6517-Snapshot
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 106b193c2ae9e111ba273b1f9dcb1c1a6e9057cb
Merge: aea5edfe99 6389d01605
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Tue Jan 31 15:05:46 2023 -0800
Merge remote-tracking branch 'asf/master' into HDDS-6517-Snapshot
Conflicts:
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
pom.xml
Change-Id: Ia2415f4046f917ac3b5f911f12fda8f6f372a8d9
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 3 +-
.../org/apache/hadoop/hdds/client/BlockID.java | 5 +
.../hadoop/hdds/client/ContainerBlockID.java | 3 +
.../hdds/client/DefaultReplicationConfig.java | 94 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 5 +-
.../java/org/apache/hadoop/hdds/utils/IOUtils.java | 17 +
.../common/src/main/resources/ozone-default.xml | 16 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 2 +-
.../container/common/helpers/ContainerUtils.java | 20 +
.../container/common/impl/ContainerDataYaml.java | 8 +-
.../common/interfaces/ContainerPacker.java | 3 +-
.../common/statemachine/DatanodeStateMachine.java | 65 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 13 +-
.../ReconstructECContainersCommandHandler.java | 19 +-
.../ReplicateContainerCommandHandler.java | 24 +-
.../container/common/volume/StorageVolume.java | 4 +
.../ECReconstructionCoordinatorTask.java | 66 +-
.../reconstruction/ECReconstructionSupervisor.java | 111 -
.../container/ec/reconstruction/TokenHelper.java | 16 +-
.../container/keyvalue/KeyValueContainer.java | 86 +-
.../container/keyvalue/KeyValueContainerCheck.java | 10 +-
.../container/keyvalue/KeyValueContainerData.java | 27 +-
.../KeyValueContainerMetadataInspector.java | 14 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +-
.../container/keyvalue/TarContainerPacker.java | 146 +-
.../helpers/KeyValueContainerLocationUtil.java | 41 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 10 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 14 +-
.../background/BlockDeletingService.java | 7 +-
.../container/ozoneimpl/ContainerController.java | 3 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 6 +-
.../replication/AbstractReplicationTask.java | 123 +
.../container/replication/ContainerDownloader.java | 2 +-
.../container/replication/ContainerImporter.java | 126 +
.../container/replication/ContainerUploader.java} | 22 +-
.../replication/CopyContainerResponseStream.java | 47 +
.../replication/DownloadAndImportReplicator.java | 101 +-
.../replication/GrpcContainerUploader.java | 100 +
.../container/replication/GrpcOutputStream.java | 41 +-
.../replication/GrpcReplicationClient.java | 21 +-
.../replication/GrpcReplicationService.java | 20 +-
.../container/replication/MeasuredReplicator.java | 23 +-
.../container/replication/PushReplicator.java | 76 +
.../container/replication/ReplicationServer.java | 7 +-
.../replication/ReplicationSupervisor.java | 170 +-
.../replication/ReplicationSupervisorMetrics.java | 7 +-
.../container/replication/ReplicationTask.java | 88 +-
.../replication/SendContainerOutputStream.java | 43 +
.../replication/SendContainerRequestHandler.java | 131 +
.../replication/SimpleContainerDownloader.java | 34 +-
.../commands/ReplicateContainerCommand.java | 76 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 2 +-
.../container/common/TestBlockDeletingService.java | 19 +-
.../TestSchemaOneBackwardsCompatibility.java | 27 +-
.../TestSchemaTwoBackwardsCompatibility.java | 4 +-
.../common/helpers/TestContainerUtils.java | 10 +
.../common/statemachine/TestStateContext.java | 9 +-
.../TestECReconstructionSupervisor.java | 158 -
.../container/keyvalue/TestKeyValueContainer.java | 10 +-
.../TestKeyValueContainerIntegrityChecks.java | 4 +-
.../TestKeyValueContainerMetadataInspector.java | 8 +-
.../container/keyvalue/TestTarContainerPacker.java | 77 +-
.../container/ozoneimpl/TestContainerReader.java | 13 +-
.../container/ozoneimpl/TestOzoneContainer.java | 7 +-
...OutputStream.java => GrpcOutputStreamTest.java} | 46 +-
.../ReplicationSupervisorScheduling.java | 85 +-
.../TestCopyContainerResponseStream.java | 50 +
.../replication/TestMeasuredReplicator.java | 32 +-
.../container/replication/TestPushReplicator.java | 140 +
.../replication/TestReplicationSupervisor.java | 242 +-
.../replication/TestSendContainerOutputStream.java | 50 +
.../replication/TestSimpleContainerDownloader.java | 50 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
hadoop-hdds/docs/content/design/trash.md | 5 -
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
hadoop-hdds/docs/content/feature/PrefixFSO.zh.md | 4 +-
.../hadoop/hdds/security/OzoneSecretKey.java | 9 +-
.../hadoop/hdds/security/OzoneSecretManager.java | 54 +-
.../security/ssl/PemFileBasedKeyStoresFactory.java | 2 +-
.../token/ContainerTokenSecretManager.java | 5 +-
.../token/OzoneBlockTokenSecretManager.java | 5 +-
.../token/ShortLivedTokenSecretManager.java | 9 +-
.../x509/certificate/authority/BaseApprover.java | 4 +-
.../certificate/authority/DefaultApprover.java | 2 +-
.../certificate/authority/DefaultCAServer.java | 10 +-
.../{PKIProfiles => profile}/DefaultCAProfile.java | 2 +-
.../{PKIProfiles => profile}/DefaultProfile.java | 2 +-
.../{PKIProfiles => profile}/PKIProfile.java | 2 +-
.../{PKIProfiles => profile}/package-info.java | 2 +-
.../x509/certificate/client/CertificateClient.java | 10 +-
.../client/CertificateNotification.java} | 19 +-
.../client/CommonCertificateClient.java | 4 +-
.../certificate/client/DNCertificateClient.java | 8 +-
.../client/DefaultCertificateClient.java | 44 +-
.../certificate/client/ReconCertificateClient.java | 8 +-
.../certificate/client/SCMCertificateClient.java | 4 +-
.../utils/CertificateSignRequest.java | 4 +-
.../utils/SelfSignedCertificate.java | 4 +-
.../CertificateException.java | 2 +-
.../{exceptions => exception}/package-info.java | 2 +-
.../hdds/security/x509/keys/SecurityUtil.java | 2 +-
.../hadoop/hdds/utils/DBCheckpointServlet.java | 69 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 18 +-
.../token/TestOzoneBlockTokenSecretManager.java | 2 +-
.../hdds/security/token/TokenVerifierTests.java | 2 +-
.../hdds/security/x509/CertificateClientTest.java | 9 +-
.../x509/certificate/authority/MockApprover.java | 2 +-
.../certificate/authority/TestDefaultCAServer.java | 10 +-
.../certificate/authority/TestDefaultProfile.java | 4 +-
.../client/TestDefaultCertificateClient.java | 4 +-
.../x509/certificate/utils/TestCRLCodec.java | 1 -
.../certificate/utils/TestCertificateCodec.java | 1 -
.../utils}/TestCertificateSignRequest.java | 3 +-
.../utils}/TestRootCertificate.java | 8 +-
.../src/main/proto/DatanodeClientProtocol.proto | 11 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 7 +
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 20 -
.../hdds/scm/container/ContainerReportHandler.java | 6 +-
.../AbstractOverReplicationHandler.java | 34 +-
.../replication/ContainerHealthResult.java | 21 +-
.../replication/ContainerReplicaCount.java | 4 +-
.../replication/ECContainerReplicaCount.java | 15 +-
.../replication/ECMisReplicationHandler.java | 16 +-
.../replication/ECOverReplicationHandler.java | 132 +-
.../replication/ECUnderReplicationHandler.java | 60 +-
.../LegacyRatisContainerReplicaCount.java | 51 +
.../replication/LegacyReplicationManager.java | 831 +++--
.../replication/MisReplicationHandler.java | 36 +-
.../replication/OverReplicatedProcessor.java | 5 +-
.../replication/RatisContainerReplicaCount.java | 170 +-
.../replication/RatisMisReplicationHandler.java | 12 +-
.../replication/RatisOverReplicationHandler.java | 66 +-
.../replication/RatisUnderReplicationHandler.java | 58 +-
.../container/replication/ReplicationManager.java | 51 +-
.../replication/UnderReplicatedProcessor.java | 5 +-
.../replication/UnhealthyReplicationHandler.java | 5 +-
.../replication/UnhealthyReplicationProcessor.java | 13 +-
.../health/ClosedWithUnhealthyReplicasHandler.java | 2 +-
.../health/DeletingContainerHandler.java | 3 +-
.../replication/health/EmptyContainerHandler.java | 2 +-
...Handler.java => MismatchedReplicasHandler.java} | 44 +-
.../health/RatisReplicationCheckHandler.java | 82 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 8 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 8 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 3 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 14 +-
.../hdds/scm/server/StorageContainerManager.java | 28 +-
.../container/replication/ReplicationTestUtil.java | 10 +
.../replication/TestECMisReplicationHandler.java | 3 +-
.../replication/TestECOverReplicationHandler.java | 130 +-
.../replication/TestECUnderReplicationHandler.java | 62 +-
.../replication/TestLegacyReplicationManager.java | 3586 +++++++++++---------
.../replication/TestMisReplicationHandler.java | 14 +-
.../replication/TestOverReplicatedProcessor.java | 9 +-
.../TestRatisContainerReplicaCount.java | 46 +
.../TestRatisMisReplicationHandler.java | 3 +-
.../TestRatisOverReplicationHandler.java | 76 +-
.../TestRatisUnderReplicationHandler.java | 14 +-
.../replication/TestReplicationManager.java | 69 +-
.../replication/TestUnderReplicatedProcessor.java | 17 +-
.../TestClosedWithUnhealthyReplicasHandler.java | 2 +-
.../health/TestDeletingContainerHandler.java | 2 +-
.../health/TestEmptyContainerHandler.java | 2 +-
...ler.java => TestMismatchedReplicasHandler.java} | 56 +-
.../health/TestRatisReplicationCheckHandler.java | 131 +
.../org/apache/ozone/test/SpyInputStream.java} | 37 +-
.../org/apache/ozone/test/SpyOutputStream.java} | 37 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 7 +-
.../client/checksum/ECFileChecksumHelper.java | 19 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 12 +-
.../hadoop/ozone/client/MockOmTransport.java | 22 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 11 +-
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 20 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 4 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 18 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 -
.../ozone/security/OzoneTokenIdentifier.java | 3 +-
.../hadoop/ozone/om/helpers/TestOmBucketArgs.java | 2 +-
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 11 +-
.../src/main/compose/ozonesecure/docker-config | 7 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 3 -
hadoop-ozone/dist/src/main/license/jar-report.txt | 5 -
.../dist/src/main/smoketest/createbucketenv.robot | 2 +-
.../dist/src/main/smoketest/createmrenv.robot | 2 +-
.../main/smoketest/debug/ozone-debug-tests.robot | 2 +-
.../dist/src/main/smoketest/gdpr/gdpr.robot | 8 +-
.../dist/src/main/smoketest/ozonefs/setup.robot | 12 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 8 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 1 +
.../main/smoketest/security/ozone-secure-fs.robot | 51 +
.../smoketest/security/ozone-secure-tenant.robot | 7 +
hadoop-ozone/integration-test/pom.xml | 5 -
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 3 +-
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 2 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 156 +-
.../hdds/scm/TestSCMDatanodeProtocolServer.java | 4 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 2 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 8 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 -
.../hadoop/ozone/TestSecureOzoneCluster.java | 187 +-
.../ozone/TestStorageContainerManagerHelper.java | 2 +-
.../ozone/client/CertificateClientTestImpl.java | 104 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 5 +-
.../client/rpc/TestFailureHandlingByClient.java | 4 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 2 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 74 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 3 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 9 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 2 +-
.../ozone/container/TestECContainerRecovery.java | 30 +-
.../commandhandler/TestBlockDeletion.java | 7 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 9 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 4 +-
.../server/TestSecureContainerServer.java | 6 +-
.../hadoop/ozone/fsck/TestContainerMapper.java | 8 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 58 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 24 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 4 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 6 +
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 4 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 14 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 8 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 58 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 64 +-
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 184 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 2 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 25 +-
.../hadoop/ozone/om/OzoneListStatusHelper.java | 49 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 76 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 22 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 6 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 14 +-
.../ozone/om/lock/RegularBucketLockStrategy.java | 20 +-
.../BucketLayoutAwareOMKeyRequestFactory.java | 2 +-
.../om/request/bucket/OMBucketCreateRequest.java | 123 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 10 +-
.../ozone/om/request/file/OMFileRequest.java | 15 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 34 -
.../om/request/volume/OMVolumeSetQuotaRequest.java | 16 +
.../S3InitiateMultipartUploadResponseWithFSO.java | 3 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +-
.../hadoop/ozone/security/OMCertificateClient.java | 6 +-
.../OzoneDelegationTokenSecretManager.java | 17 +-
.../org/apache/hadoop/ozone/om/TestOMStorage.java | 315 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 14 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 4 +
.../request/bucket/TestOMBucketCreateRequest.java | 26 +-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 4 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 53 +-
.../om/request/file/TestOMFileCreateRequest.java | 23 -
.../om/request/key/TestOMAllocateBlockRequest.java | 24 -
.../om/request/key/TestOMKeyCommitRequest.java | 71 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 11 +-
.../om/request/key/TestOMKeyCreateRequest.java | 23 -
.../volume/TestOMVolumeSetQuotaRequest.java | 23 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 2 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 44 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 35 +-
.../org/apache/hadoop/fs/ozone/TestOFSPath.java | 26 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 20 +
hadoop-ozone/pom.xml | 6 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 24 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 6 +-
.../recon/api/handlers/BucketEntityHandler.java | 36 +-
.../ozone/recon/api/handlers/BucketHandler.java | 4 +
.../recon/api/handlers/DirectoryEntityHandler.java | 26 +-
.../ozone/recon/api/handlers/FSOBucketHandler.java | 20 +
.../ozone/recon/api/handlers/KeyEntityHandler.java | 23 +-
.../recon/api/handlers/LegacyBucketHandler.java | 14 +
.../recon/api/handlers/RootEntityHandler.java | 30 +-
.../recon/api/handlers/UnknownEntityHandler.java | 8 +-
.../recon/api/handlers/VolumeEntityHandler.java | 32 +-
.../ozone/recon/api/types/BucketObjectDBInfo.java | 168 +
...mespaceSummaryResponse.java => CountStats.java} | 70 +-
.../ozone/recon/api/types/KeyObjectDBInfo.java | 154 +
.../recon/api/types/NamespaceSummaryResponse.java | 138 +-
.../hadoop/ozone/recon/api/types/ObjectDBInfo.java | 140 +
.../ozone/recon/api/types/VolumeObjectDBInfo.java | 78 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 2 +-
.../impl/StorageContainerServiceProviderImpl.java | 2 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 15 +-
.../webapps/recon/ozone-recon-web/api/db.json | 135 +-
.../src/views/diskUsage/diskUsage.tsx | 131 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 61 +-
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 62 +-
.../hadoop/ozone/recon/common/CommonUtils.java | 225 ++
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 12 +-
hadoop-ozone/tools/pom.xml | 5 +-
.../ozone/admin/nssummary/NSSummaryAdmin.java | 9 +-
.../ozone/debug/container/ExportSubcommand.java | 2 +-
.../ozone/freon/ClosedContainerReplicator.java | 20 +-
pom.xml | 139 +-
294 files changed, 9290 insertions(+), 4714 deletions(-)
diff --cc hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
index 8e372ffc3c,0000000000..842994cc28
mode 100644,000000..100644
--- a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
+++ b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
@@@ -1,213 -1,0 +1,193 @@@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds</artifactId>
+ <version>1.4.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>rocksdb-checkpoint-differ</artifactId>
+ <version>1.4.0-SNAPSHOT</version>
+ <description>RocksDB Checkpoint Differ</description>
+ <name>RocksDB Checkpoint Differ</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds-common</artifactId>
+ <version>${hdds.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
- <dependency>
- <groupId>com.github.vlsi.mxgraph</groupId>
- <artifactId>jgraphx</artifactId>
- <version>4.2.2</version>
- </dependency>
- <dependency>
- <groupId>org.jgrapht</groupId>
- <artifactId>jgrapht-core</artifactId>
- <version>1.5.0</version>
- </dependency>
- <dependency>
- <groupId>org.jgrapht</groupId>
- <artifactId>jgrapht-guava</artifactId>
- <version>1.5.0</version>
- </dependency>
- <dependency>
- <groupId>org.jgrapht</groupId>
- <artifactId>jgrapht-ext</artifactId>
- <version>1.4.0</version>
- </dependency>
+ <dependency>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-annotations</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds-test-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <excludes>
+ <exclude>ozone-version-info.properties</exclude>
+ </excludes>
+ <filtering>false</filtering>
+ </resource>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>ozone-version-info.properties</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>version-info</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>version-info</goal>
+ </goals>
+ <configuration>
+ <source>
+ <directory>${basedir}/../</directory>
+ <includes>
+ <include>*/src/main/java/**/*.java</include>
+ <include>*/src/main/proto/*.proto</include>
+ </includes>
+ </source>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>depcheck</id>
+ <phase></phase>
+ </execution>
+ <execution>
+ <id>banned-rocksdb-imports</id>
+ <phase>process-sources</phase>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <RestrictImports>
+ <includeTestCode>false</includeTestCode>
+ <reason>Use managed RocksObjects under org.apache.hadoop.hdds.utils.db.managed instead.</reason>
+ <!-- By default, ban all the classes in org.rocksdb -->
+ <bannedImport>org.rocksdb.**</bannedImport>
+ <allowedImports>
+ <allowedImport>org.rocksdb.AbstractEventListener</allowedImport>
+ <allowedImport>org.rocksdb.Checkpoint</allowedImport>
+ <allowedImport>org.rocksdb.ColumnFamilyDescriptor</allowedImport>
+ <allowedImport>org.rocksdb.ColumnFamilyHandle</allowedImport>
+ <allowedImport>org.rocksdb.ColumnFamilyOptions</allowedImport>
+ <allowedImport>org.rocksdb.CompactionJobInfo</allowedImport>
+ <allowedImport>org.rocksdb.CompressionType</allowedImport>
+ <allowedImport>org.rocksdb.DBOptions</allowedImport>
+ <allowedImport>org.rocksdb.FlushOptions</allowedImport>
+ <allowedImport>org.rocksdb.LiveFileMetaData</allowedImport>
+ <allowedImport>org.rocksdb.Options</allowedImport>
+ <allowedImport>org.rocksdb.RocksDB</allowedImport>
+ <allowedImport>org.rocksdb.RocksDBException</allowedImport>
+ <allowedImport>org.rocksdb.SstFileReader</allowedImport>
+ <allowedImport>org.rocksdb.TableProperties</allowedImport>
+ <allowedImport>org.rocksdb.ReadOptions</allowedImport>
+ <allowedImport>org.rocksdb.SstFileReaderIterator</allowedImport>
+ </allowedImports>
+ <exclusion>org.apache.hadoop.hdds.utils.db.managed.*</exclusion>
+ </RestrictImports>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <profiles>
+ <profile>
+ <id>k8s-dev</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>io.fabric8</groupId>
+ <artifactId>docker-maven-plugin</artifactId>
+ <configuration>
+ <images>
+ <image>
+ <name>${user.name}/ozone:${project.version}</name>
+ <build>
+ <dockerFileDir>${project.basedir}</dockerFileDir>
+ </build>
+ </image>
+ </images>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --cc hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
index 83ae64aaea,828dc7e27a..f62e2cf563
--- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
+++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
@@@ -460,25 -459,6 +460,22 @@@ MI
org.slf4j:slf4j-reload4j
+EPL 2.0
+=====================
+
+ jakarta.annotation:jakarta.annotation-api
+ jakarta.ws.rs:jakarta.ws.rs-api
- org.jgrapht:jgrapht-core
- org.jgrapht:jgrapht-guava
- org.jgrapht:jgrapht-ext
+
+
+CDDL + GPLv2 with classpath exception
+=====================
+
+ javax.annotation:javax.annotation-api
+ javax.el:javax.el-api
+ javax.interceptor:javax.interceptor-api
+ javax.servlet:javax.servlet-api
+
+
Public Domain
=====================
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index 06b8e48b6a,0000000000..e8258adfa4
mode 100644,000000..100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@@ -1,583 -1,0 +1,583 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import java.io.IOException;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.server.OzoneAdmins;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.audit.AuditAction;
+import org.apache.hadoop.ozone.audit.AuditEventStatus;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.AuditMessage;
+import org.apache.hadoop.ozone.audit.Auditor;
+import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
+import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
+import org.apache.hadoop.ozone.security.acl.RequestContext;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import java.net.InetAddress;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE_MAX;
+import static org.apache.hadoop.ozone.om.KeyManagerImpl.getRemoteUser;
+import static org.apache.hadoop.ozone.om.OzoneManager.getS3Auth;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
+import org.apache.hadoop.ozone.security.acl.OzoneAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType;
+import org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
+import static org.apache.hadoop.util.MetricUtil.captureLatencyNs;
+
+/**
+ * OM Metadata Reading class for the OM and Snapshot managers.
+ *
+ * This abstraction manages all the metadata key/acl reading
+ * from a rocksDb instance, for both the OM and OM snapshots.
+ */
+public class OmMetadataReader implements IOmMetadataReader, Auditor {
+ private final KeyManager keyManager;
+ private final PrefixManager prefixManager;
+ private final VolumeManager volumeManager;
+ private final BucketManager bucketManager;
+ private final OzoneManager ozoneManager;
+ private final boolean isAclEnabled;
+ private final IAccessAuthorizer accessAuthorizer;
+ private final boolean isNativeAuthorizerEnabled;
+ private final OmMetadataReaderMetrics metrics;
+ private final Logger log;
+ private final AuditLogger audit;
+ private final OMPerformanceMetrics perfMetrics;
+
+ public OmMetadataReader(KeyManager keyManager,
+ PrefixManager prefixManager,
+ OzoneManager ozoneManager,
+ Logger log,
+ AuditLogger audit,
+ OmMetadataReaderMetrics omMetadataReaderMetrics) {
+ this.keyManager = keyManager;
+ this.bucketManager = ozoneManager.getBucketManager();
+ this.volumeManager = ozoneManager.getVolumeManager();
+ this.prefixManager = prefixManager;
+ OzoneConfiguration configuration = ozoneManager.getConfiguration();
+ this.ozoneManager = ozoneManager;
+ this.isAclEnabled = ozoneManager.getAclsEnabled();
+ this.log = log;
+ this.audit = audit;
+ boolean allowListAllVolumes = ozoneManager.getAllowListAllVolumes();
+ this.metrics = omMetadataReaderMetrics;
+ this.perfMetrics = ozoneManager.getPerfMetrics();
+ if (isAclEnabled) {
+ accessAuthorizer = getACLAuthorizerInstance(configuration);
+ if (accessAuthorizer instanceof OzoneNativeAuthorizer) {
+ OzoneNativeAuthorizer authorizer =
+ (OzoneNativeAuthorizer) accessAuthorizer;
+ isNativeAuthorizerEnabled = true;
+ authorizer.setVolumeManager(volumeManager);
+ authorizer.setBucketManager(bucketManager);
+ authorizer.setKeyManager(keyManager);
+ authorizer.setPrefixManager(prefixManager);
+ authorizer.setOzoneAdmins(
+ new OzoneAdmins(ozoneManager.getOmAdminUsernames()));
+ authorizer.setAllowListAllVolumes(allowListAllVolumes);
+ } else {
+ isNativeAuthorizerEnabled = false;
+ }
+ } else {
+ accessAuthorizer = null;
+ isNativeAuthorizerEnabled = false;
+ }
+ }
+
+ /**
+ * Lookup a key.
+ *
+ * @param args - attributes of the key.
+ * @return OmKeyInfo - the info about the requested key.
+ * @throws IOException
+ */
+ @Override
+ public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException {
+ long start = Time.monotonicNowNanos();
+ ResolvedBucket bucket = captureLatencyNs(
+ perfMetrics.getLookupResolveBucketLatencyNs(),
+ () -> ozoneManager.resolveBucketLink(args));
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit(args.toAuditMap());
+
+ OmKeyArgs resolvedArgs = bucket.update(args);
+
+ try {
+ if (isAclEnabled) {
+ captureLatencyNs(perfMetrics.getLookupAclCheckLatencyNs(),
+ () -> checkAcls(ResourceType.KEY, StoreType.OZONE,
+ ACLType.READ, bucket.realVolume(), bucket.realBucket(),
+ args.getKeyName())
+ );
+ }
+ metrics.incNumKeyLookups();
+ return keyManager.lookupKey(resolvedArgs, getClientAddress());
+ } catch (Exception ex) {
+ metrics.incNumKeyLookupFails();
+ auditSuccess = false;
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.READ_KEY,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.READ_KEY,
+ auditMap));
+ }
+
+ perfMetrics.addLookupLatency(Time.monotonicNowNanos() - start);
+ }
+ }
+
+ @Override
+ public KeyInfoWithVolumeContext getKeyInfo(final OmKeyArgs args,
+ boolean assumeS3Context)
+ throws IOException {
+ long start = Time.monotonicNowNanos();
+
+ java.util.Optional<S3VolumeContext> s3VolumeContext =
+ java.util.Optional.empty();
+
+ final OmKeyArgs resolvedVolumeArgs;
+ if (assumeS3Context) {
+ S3VolumeContext context = ozoneManager.getS3VolumeContext();
+ s3VolumeContext = java.util.Optional.of(context);
+ resolvedVolumeArgs = args.toBuilder()
+ .setVolumeName(context.getOmVolumeArgs().getVolume())
+ .build();
+ } else {
+ resolvedVolumeArgs = args;
+ }
+
+ final ResolvedBucket bucket = captureLatencyNs(
+ perfMetrics.getGetKeyInfoResolveBucketLatencyNs(),
+ () -> ozoneManager.resolveBucketLink(resolvedVolumeArgs));
+
+ boolean auditSuccess = true;
- OmKeyArgs resolvedArgs = bucket.update(args);
++ OmKeyArgs resolvedArgs = bucket.update(resolvedVolumeArgs);
+
+ try {
+ if (isAclEnabled) {
+ captureLatencyNs(perfMetrics.getGetKeyInfoAclCheckLatencyNs(), () ->
+ checkAcls(ResourceType.KEY,
+ StoreType.OZONE, ACLType.READ,
+ bucket.realVolume(), bucket.realBucket(), args.getKeyName())
+ );
+ }
+
+ metrics.incNumGetKeyInfo();
+ OmKeyInfo keyInfo =
+ keyManager.getKeyInfo(resolvedArgs,
+ OmMetadataReader.getClientAddress());
+ KeyInfoWithVolumeContext.Builder builder = KeyInfoWithVolumeContext
+ .newBuilder()
+ .setKeyInfo(keyInfo);
+ s3VolumeContext.ifPresent(context -> {
+ builder.setVolumeArgs(context.getOmVolumeArgs());
+ builder.setUserPrincipal(context.getUserPrincipal());
+ });
+ return builder.build();
+ } catch (Exception ex) {
+ metrics.incNumGetKeyInfoFails();
+ auditSuccess = false;
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.READ_KEY,
+ bucket.audit(resolvedVolumeArgs.toAuditMap()), ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.READ_KEY,
+ bucket.audit(resolvedVolumeArgs.toAuditMap())));
+ }
+ perfMetrics.addGetKeyInfoLatencyNs(Time.monotonicNowNanos() - start);
+ }
+ }
+
+ @Override
+ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
+ String startKey, long numEntries, boolean allowPartialPrefixes)
+ throws IOException {
+
+ long maxListingPageSize = ozoneManager.getConfiguration().getInt(
+ OZONE_FS_LISTING_PAGE_SIZE_MAX,
+ OZONE_FS_LISTING_PAGE_SIZE_DEFAULT);
+ maxListingPageSize = OzoneConfigUtil.limitValue(numEntries,
+ OZONE_FS_LISTING_PAGE_SIZE, OZONE_FS_LISTING_PAGE_SIZE_MAX,
+ maxListingPageSize);
+
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
+
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit(args.toAuditMap());
+
+ args = bucket.update(args);
+
+ try {
+ if (isAclEnabled) {
+ checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ,
+ bucket.realVolume(), bucket.realBucket(), args.getKeyName());
+ }
+ metrics.incNumListStatus();
+ return keyManager.listStatus(args, recursive, startKey,
+ maxListingPageSize, getClientAddress(), allowPartialPrefixes);
+ } catch (Exception ex) {
+ metrics.incNumListStatusFails();
+ auditSuccess = false;
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_STATUS,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(buildAuditMessageForSuccess(
+ OMAction.LIST_STATUS, auditMap));
+ }
+ }
+ }
+
+ @Override
+ public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException {
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
+
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit(args.toAuditMap());
+
+ args = bucket.update(args);
+
+ try {
+ metrics.incNumGetFileStatus();
+ return keyManager.getFileStatus(args, getClientAddress());
+ } catch (IOException ex) {
+ metrics.incNumGetFileStatusFails();
+ auditSuccess = false;
+ audit.logReadFailure(
+ buildAuditMessageForFailure(OMAction.GET_FILE_STATUS, auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(
+ buildAuditMessageForSuccess(OMAction.GET_FILE_STATUS, auditMap));
+ }
+ }
+ }
+
+ @Override
+ public OmKeyInfo lookupFile(OmKeyArgs args) throws IOException {
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(args);
+
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit(args.toAuditMap());
+
+ args = bucket.update(args);
+
+ try {
+ if (isAclEnabled) {
+ checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.READ,
+ bucket.realVolume(), bucket.realBucket(), args.getKeyName());
+ }
+ metrics.incNumLookupFile();
+ return keyManager.lookupFile(args, getClientAddress());
+ } catch (Exception ex) {
+ metrics.incNumLookupFileFails();
+ auditSuccess = false;
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.LOOKUP_FILE,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(buildAuditMessageForSuccess(
+ OMAction.LOOKUP_FILE, auditMap));
+ }
+ }
+ }
+
+ @Override
+ public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
+ String startKey, String keyPrefix, int maxKeys) throws IOException {
+
+ ResolvedBucket bucket = ozoneManager.resolveBucketLink(
+ Pair.of(volumeName, bucketName));
+
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = bucket.audit();
+ auditMap.put(OzoneConsts.START_KEY, startKey);
+ auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys));
+ auditMap.put(OzoneConsts.KEY_PREFIX, keyPrefix);
+
+ try {
+ if (isAclEnabled) {
+ checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST,
+ bucket.realVolume(), bucket.realBucket(), keyPrefix);
+ }
+ metrics.incNumKeyLists();
+ return keyManager.listKeys(bucket.realVolume(), bucket.realBucket(),
+ startKey, keyPrefix, maxKeys);
+ } catch (IOException ex) {
+ metrics.incNumKeyListFails();
+ auditSuccess = false;
+ audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_KEYS,
+ auditMap, ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_KEYS,
+ auditMap));
+ }
+ }
+ }
+
+ /**
+ * Returns list of ACLs for given Ozone object.
+ *
+ * @param obj Ozone object.
+ * @throws IOException if there is error.
+ */
+ public List<OzoneAcl> getAcl(OzoneObj obj) throws IOException {
+ boolean auditSuccess = true;
+
+ try {
+ if (isAclEnabled) {
+ checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.READ_ACL,
+ obj.getVolumeName(), obj.getBucketName(), obj.getKeyName());
+ }
+ metrics.incNumGetAcl();
+ switch (obj.getResourceType()) {
+ case VOLUME:
+ return volumeManager.getAcl(obj);
+ case BUCKET:
+ return bucketManager.getAcl(obj);
+ case KEY:
+ return keyManager.getAcl(obj);
+ case PREFIX:
+ return prefixManager.getAcl(obj);
+
+ default:
+ throw new OMException("Unexpected resource type: " +
+ obj.getResourceType(), INVALID_REQUEST);
+ }
+ } catch (Exception ex) {
+ auditSuccess = false;
+ audit.logReadFailure(
+ buildAuditMessageForFailure(OMAction.GET_ACL, obj.toAuditMap(), ex));
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ audit.logReadSuccess(
+ buildAuditMessageForSuccess(OMAction.GET_ACL, obj.toAuditMap()));
+ }
+ }
+ }
+
+ /**
+ * Checks if current caller has acl permissions.
+ *
+ * @param resType - Type of ozone resource. Ex volume, bucket.
+ * @param store - Store type. i.e Ozone, S3.
+ * @param acl - type of access to be checked.
+ * @param vol - name of volume
+ * @param bucket - bucket name
+ * @param key - key
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied.
+ */
+ void checkAcls(ResourceType resType, StoreType store,
+ ACLType acl, String vol, String bucket, String key)
+ throws IOException {
+ UserGroupInformation user;
+ if (getS3Auth() != null) {
+ String principal =
+ OzoneAclUtils.accessIdToUserPrincipal(getS3Auth().getAccessId());
+ user = UserGroupInformation.createRemoteUser(principal);
+ } else {
+ user = ProtobufRpcEngine.Server.getRemoteUser();
+ }
+
+ InetAddress remoteIp = ProtobufRpcEngine.Server.getRemoteIp();
+ String volumeOwner = ozoneManager.getVolumeOwner(vol, acl, resType);
+ String bucketOwner = ozoneManager.getBucketOwner(vol, bucket, acl, resType);
+
+ OzoneAclUtils.checkAllAcls(this, resType, store, acl,
+ vol, bucket, key, volumeOwner, bucketOwner,
+ user != null ? user : getRemoteUser(),
+ remoteIp != null ? remoteIp :
+ ozoneManager.getOmRpcServerAddr().getAddress(),
+ remoteIp != null ? remoteIp.getHostName() :
+ ozoneManager.getOmRpcServerAddr().getHostName());
+ }
+
+
+ /**
+ * CheckAcls for the ozone object.
+ *
+ * @return true if permission granted, false if permission denied.
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied
+ * and throwOnPermissionDenied set to true.
+ */
+ @SuppressWarnings("parameternumber")
+ public boolean checkAcls(ResourceType resType, StoreType storeType,
+ ACLType aclType, String vol, String bucket, String key,
+ UserGroupInformation ugi, InetAddress remoteAddress, String hostName,
+ boolean throwIfPermissionDenied, String owner)
+ throws OMException {
+ OzoneObj obj = OzoneObjInfo.Builder.newBuilder()
+ .setResType(resType)
+ .setStoreType(storeType)
+ .setVolumeName(vol)
+ .setBucketName(bucket)
+ .setKeyName(key).build();
+ RequestContext context = RequestContext.newBuilder()
+ .setClientUgi(ugi)
+ .setIp(remoteAddress)
+ .setHost(hostName)
+ .setAclType(ACLIdentityType.USER)
+ .setAclRights(aclType)
+ .setOwnerName(owner)
+ .build();
+
+ return checkAcls(obj, context, throwIfPermissionDenied);
+ }
+
+ /**
+ * CheckAcls for the ozone object.
+ *
+ * @return true if permission granted, false if permission denied.
+ * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied
+ * and throwOnPermissionDenied set to true.
+ */
+ public boolean checkAcls(OzoneObj obj, RequestContext context,
+ boolean throwIfPermissionDenied)
+ throws OMException {
+
+ if (!accessAuthorizer.checkAccess(obj, context)) {
+ if (throwIfPermissionDenied) {
+ String volumeName = obj.getVolumeName() != null ?
+ "Volume:" + obj.getVolumeName() + " " : "";
+ String bucketName = obj.getBucketName() != null ?
+ "Bucket:" + obj.getBucketName() + " " : "";
+ String keyName = obj.getKeyName() != null ?
+ "Key:" + obj.getKeyName() : "";
+ log.warn("User {} doesn't have {} permission to access {} {}{}{}",
+ context.getClientUgi().getUserName(), context.getAclRights(),
+ obj.getResourceType(), volumeName, bucketName, keyName);
+ throw new OMException("User " + context.getClientUgi().getUserName() +
+ " doesn't have " + context.getAclRights() +
+ " permission to access " + obj.getResourceType() + " " +
+ volumeName + bucketName + keyName, ResultCodes.PERMISSION_DENIED);
+ }
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * Returns an instance of {@link IAccessAuthorizer}.
+ * Looks up the configuration to see if there is custom class specified.
+ * Constructs the instance by passing the configuration directly to the
+ * constructor to achieve thread safety using final fields.
+ *
+ * @param conf
+ * @return IAccessAuthorizer
+ */
+ private IAccessAuthorizer getACLAuthorizerInstance(OzoneConfiguration conf) {
+ Class<? extends IAccessAuthorizer> clazz = conf.getClass(
+ OZONE_ACL_AUTHORIZER_CLASS, OzoneAccessAuthorizer.class,
+ IAccessAuthorizer.class);
+ return ReflectionUtils.newInstance(clazz, conf);
+ }
+
+ static String getClientAddress() {
+ String clientMachine = Server.getRemoteAddress();
+ if (clientMachine == null) { //not a RPC client
+ clientMachine = "";
+ }
+ return clientMachine;
+ }
+
+ @Override
+ public AuditMessage buildAuditMessageForSuccess(AuditAction op,
+ Map<String, String> auditMap) {
+
+ return new AuditMessage.Builder()
+ .setUser(getRemoteUserName())
+ .atIp(Server.getRemoteAddress())
+ .forOperation(op)
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.SUCCESS)
+ .build();
+ }
+
+ @Override
+ public AuditMessage buildAuditMessageForFailure(AuditAction op,
+ Map<String, String> auditMap, Throwable throwable) {
+
+ return new AuditMessage.Builder()
+ .setUser(getRemoteUserName())
+ .atIp(Server.getRemoteAddress())
+ .forOperation(op)
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.FAILURE)
+ .withException(throwable)
+ .build();
+ }
+
+ /**
+ * Returns true if OzoneNativeAuthorizer is enabled and false if otherwise.
+ *
+ * @return if native authorizer is enabled.
+ */
+ public boolean isNativeAuthorizerEnabled() {
+ return isNativeAuthorizerEnabled;
+ }
+
+ public IAccessAuthorizer getAccessAuthorizer() {
+ return accessAuthorizer;
+ }
+
+ private ResourceType getResourceType(OmKeyArgs args) {
+ if (args.getKeyName() == null || args.getKeyName().length() == 0) {
+ return ResourceType.BUCKET;
+ }
+ return ResourceType.KEY;
+ }
+
+
+}
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 8eef324523,15b579c9c9..169e87fa39
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@@ -71,10 -73,8 +73,11 @@@ import org.apache.hadoop.hdds.utils.db.
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.OzoneManagerVersion;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.service.OMRangerBGSyncService;
+ import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
import org.apache.hadoop.ozone.util.OzoneNetUtils;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
diff --cc hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index 9dcd365154,3eddc01d6c..f36dad9a96
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@@ -637,13 -636,4 +637,13 @@@ public class BasicOzoneClientAdapterImp
length, combineMode, ozoneClient.getObjectStore().getClientProxy());
}
+
+ @Override
+ public String createSnapshot(String pathStr, String snapshotName)
+ throws IOException {
- OFSPath ofsPath = new OFSPath(pathStr);
++ OFSPath ofsPath = new OFSPath(pathStr, config);
+ return objectStore.createSnapshot(ofsPath.getVolumeName(),
+ ofsPath.getBucketName(),
+ snapshotName);
+ }
}
diff --cc hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 6eddf06dac,2606bd4fdf..c1d882b2e4
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@@ -1157,13 -1159,4 +1159,13 @@@ public class BasicRootedOzoneClientAdap
length, combineMode, ozoneClient.getObjectStore().getClientProxy());
}
+
+ @Override
+ public String createSnapshot(String pathStr, String snapshotName)
+ throws IOException {
- OFSPath ofsPath = new OFSPath(pathStr);
++ OFSPath ofsPath = new OFSPath(pathStr, config);
+ return proxy.createSnapshot(ofsPath.getVolumeName(),
+ ofsPath.getBucketName(),
+ snapshotName);
+ }
}
diff --cc pom.xml
index 9a2bc44173,c330243178..5101bf85da
--- a/pom.xml
+++ b/pom.xml
@@@ -194,7 -198,18 +198,18 @@@ xsi:schemaLocation="http://maven.apache
<findbugs.version>3.0.0</findbugs.version>
<spotbugs.version>3.1.12</spotbugs.version>
<dnsjava.version>2.1.7</dnsjava.version>
+ <jakarta.activation.version>1.2.2</jakarta.activation.version>
+ <okhttp3.version>4.9.3</okhttp3.version>
+ <stax2.version>4.2.1</stax2.version>
+ <nimbus.version>9.8.1</nimbus.version>
+ <checker.version>3.12.0</checker.version>
+ <jakarta.inject.version>2.6.1</jakarta.inject.version>
+ <jakarta.annotation.version>1.3.5</jakarta.annotation.version>
+ <joda.time.version>2.10.6</joda.time.version>
+ <commons.lang.version>2.6</commons.lang.version>
+ <zookeeper.version>3.5.6</zookeeper.version>
+ <snappy.java.version>1.1.8.2</snappy.java.version>
-
+
<compile-testing.version>0.19</compile-testing.version>
<errorprone-annotations.version>2.2.0</errorprone-annotations.version>
<guava.version>31.1-jre</guava.version>
@@@ -1444,11 -1478,96 +1479,101 @@@
<artifactId>sqlite-jdbc</artifactId>
<version>${sqlite.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <version>${awaitility.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.activation</groupId>
+ <artifactId>jakarta.activation-api</artifactId>
+ <version>${jakarta.activation.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>okhttp</artifactId>
+ <version>${okhttp3.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.woodstox</groupId>
+ <artifactId>stax2-api</artifactId>
+ <version>${stax2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.nimbusds</groupId>
+ <artifactId>nimbus-jose-jwt</artifactId>
+ <version>${nimbus.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.checkerframework</groupId>
+ <artifactId>checker-qual</artifactId>
+ <version>${checker.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ <version>${java.dev.jna.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna-platform</artifactId>
+ <version>${java.dev.jna.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.hk2.external</groupId>
+ <artifactId>jakarta.inject</artifactId>
+ <version>${jakarta.inject.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.annotation</groupId>
+ <artifactId>jakarta.annotation-api</artifactId>
+ <version>${jakarta.annotation.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>${joda.time.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <version>${codehaus.jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ <version>${codehaus.jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ <version>${codehaus.jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons.lang.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${apache.curator}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ <version>${apache.curator}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>${snappy.java.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org