You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2022/04/12 17:51:50 UTC
[ozone] 01/01: Merge master to EC Branch HDDS-3816-ec
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
commit e9b621521095160e5422ff2c760244005a1957fd
Merge: df96b86c8d 55d5ecbda2
Author: Uma Maheswara Rao G <um...@cloudera.com>
AuthorDate: Mon Apr 11 10:52:51 2022 -0700
Merge master to EC Branch HDDS-3816-ec
.../java/org/apache/hadoop/hdds/HddsUtils.java | 1 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 10 +-
.../ContainerCommandResponseBuilders.java | 13 +-
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 35 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 49 +-
.../ozone/container/ContainerTestHelper.java | 28 +
.../common/statemachine/DatanodeStateMachine.java | 31 ++
.../common/statemachine/StateContext.java | 13 +
.../commandhandler/CommandDispatcher.java | 16 +
.../commandhandler/CommandHandler.java | 10 +
.../commandhandler/DeleteBlocksCommandHandler.java | 11 +
.../DeleteContainerCommandHandler.java | 5 +
.../ReplicateContainerCommandHandler.java | 5 +
.../states/endpoint/HeartbeatEndpointTask.java | 21 +
.../ozone/container/keyvalue/KeyValueHandler.java | 40 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 10 +-
.../replication/ReplicationSupervisor.java | 2 +-
.../container/common/impl/TestHddsDispatcher.java | 20 +
.../common/statemachine/TestStateContext.java | 36 ++
.../states/endpoint/TestHeartbeatEndpointTask.java | 26 +-
.../container/keyvalue/TestKeyValueHandler.java | 7 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 1 +
hadoop-hdds/docs/content/interface/S3.md | 7 -
hadoop-hdds/docs/content/interface/S3.zh.md | 7 -
.../docs/content/start/StartFromDockerHub.md | 6 -
.../docs/content/start/StartFromDockerHub.zh.md | 5 -
...inerLocationProtocolClientSideTranslatorPB.java | 15 +-
.../hadoop/hdds/server/http/HttpServer2.java | 1 +
.../src/main/proto/ScmAdminProtocol.proto | 1 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 6 +
.../scm/container/balancer/ContainerBalancer.java | 397 ++++++++-----
.../balancer/ContainerBalancerMetrics.java | 50 +-
.../IllegalContainerBalancerStateException.java | 46 ++
...lidContainerBalancerConfigurationException.java | 47 ++
...inerLocationProtocolServerSideTranslatorPB.java | 17 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 40 +-
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../container/balancer/TestContainerBalancer.java | 155 ++++--
.../scm/cli/ContainerBalancerStartSubcommand.java | 14 +-
.../hdds/scm/cli/ContainerOperationClient.java | 8 +-
.../datanode/TestContainerBalancerSubCommand.java | 22 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 1 +
.../ozone/client/protocol/ClientProtocol.java | 1 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 1 +
.../apache/hadoop/ozone/om/lock/LockUsageInfo.java | 63 +++
.../apache/hadoop/ozone/om/lock/OMLockMetrics.java | 207 +++++++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 273 ++++++++-
...OzoneManagerProtocolClientSideTranslatorPB.java | 1 +
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 227 ++++++++
.../dist/src/main/compose/xcompat/docker-config | 3 +-
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 3 +
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 245 ++++++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 6 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 48 ++
.../rooted/ITestRootedOzoneContractRootDir.java | 1 -
.../TestContainerStateManagerIntegration.java | 89 +--
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 4 +
.../container/ozoneimpl/TestOzoneContainer.java | 2 -
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 81 +++
.../om/ratis/TestOzoneManagerRatisRequest.java | 35 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 56 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 11 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 144 +++--
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 72 ++-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 77 ++-
.../BucketLayoutAwareOMKeyRequestFactory.java | 314 +++++++++++
.../ozone/om/request/OMClientRequestUtils.java | 50 ++
.../ozone/om/request/OMKeyRequestFactory.java | 139 -----
.../om/request/bucket/OMBucketCreateRequest.java | 6 +-
.../om/request/file/OMDirectoryCreateRequest.java | 19 -
.../ozone/om/request/file/OMFileCreateRequest.java | 17 -
.../ozone/om/request/file/OMFileRequest.java | 56 +-
.../om/request/key/OMAllocateBlockRequest.java | 14 -
.../ozone/om/request/key/OMKeyCommitRequest.java | 15 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 18 -
.../ozone/om/request/key/OMKeyDeleteRequest.java | 15 -
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 4 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 19 -
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 7 +
.../ozone/om/request/key/OMKeysDeleteRequest.java | 137 +++--
.../ozone/om/request/key/OMKeysRenameRequest.java | 5 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 6 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 141 +++++
.../S3InitiateMultipartUploadRequest.java | 15 -
.../multipart/S3MultipartUploadAbortRequest.java | 15 -
.../S3MultipartUploadCommitPartRequest.java | 15 -
.../S3MultipartUploadCompleteRequest.java | 27 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 3 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 16 -
.../om/response/key/OMKeysDeleteResponse.java | 8 +
.../response/key/OMKeysDeleteResponseWithFSO.java | 94 ++++
.../org/apache/hadoop/ozone/om/OmTestManagers.java | 12 +-
.../ozone/om/ScmBlockLocationTestingClient.java | 12 +
.../hadoop/ozone/om/TestKeyDeletingService.java | 135 +++--
.../request/TestBucketLayoutAwareOMKeyFactory.java | 166 ++++++
.../request/file/TestOMDirectoryCreateRequest.java | 37 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 16 +-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../file/TestOMFileCreateRequestWithFSO.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 7 +-
.../om/request/key/TestOMKeyRenameRequest.java | 14 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 35 +-
.../key/TestOMKeysDeleteRequestWithFSO.java | 109 ++++
.../om/request/key/TestOMKeysRenameRequest.java | 4 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 4 +-
.../TestS3InitiateMultipartUploadRequest.java | 2 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../TestS3MultipartUploadAbortRequest.java | 4 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 5 +
.../TestS3MultipartUploadCommitPartRequest.java | 6 +-
.../TestS3MultipartUploadCompleteRequest.java | 10 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 7 +-
.../upgrade/TestOMCancelPrepareRequest.java | 2 +-
.../ozone/om/response/TestCleanupTableInfo.java | 6 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 133 ++++-
hadoop-ozone/ozonefs-shaded/pom.xml | 6 +
hadoop-ozone/pom.xml | 2 +-
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 2 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 37 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 252 ++++++---
hadoop-ozone/s3gateway/pom.xml | 4 +
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 19 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 10 -
.../s3gateway/src/main/resources/browser.html | 617 ---------------------
.../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 36 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +-
pom.xml | 7 +-
129 files changed, 4035 insertions(+), 1765 deletions(-)
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index f014251109,e2a5c8c750..dd5690359e
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@@ -19,7 -19,7 +19,8 @@@ package org.apache.hadoop.hdds.scm.clie
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.container.ContainerReplicaInfo;
import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 1df588e5f5,1b07c5d3d6..619f8353fb
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@@ -18,8 -18,8 +18,9 @@@
package org.apache.hadoop.hdds.scm.protocol;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.ScmConfig;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
index d7edcea17c,223986b5ec..a7fdef0bcd
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
@@@ -29,11 -28,8 +28,10 @@@ import java.util.Map
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OzoneAcl;
- import org.apache.hadoop.ozone.om.OzoneManagerUtils;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index 0bf7f3ade2,39f090bcdd..824a78d13a
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@@ -26,8 -25,6 +25,7 @@@ import java.util.Map
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
- import org.apache.hadoop.ozone.om.OzoneManagerUtils;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 52dff6294a,26a68b89f4..8a90246d15
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@@ -51,8 -49,9 +51,9 @@@ import org.apache.hadoop.ozone.om.helpe
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
-import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.QuotaUtil;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+ import org.apache.hadoop.ozone.om.request.OMClientRequestUtils;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
index 2c5e25df36,c54faf6815..97002898e6
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java
@@@ -23,9 -23,7 +23,8 @@@ import com.google.common.base.Precondit
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneConfigUtil;
import org.apache.hadoop.ozone.om.OzoneManager;
- import org.apache.hadoop.ozone.om.OzoneManagerUtils;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
index 6823a92e28,cb5bb230c9..f0af5e7e55
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
@@@ -485,10 -424,9 +485,9 @@@ public class TestOMKeyCreateRequest ext
// Add volume and bucket entries to DB.
addVolumeAndBucketToDB(volumeName, bucketName,
- omMetadataManager);
-
+ omMetadataManager, getBucketLayout());
- keyName = "dir1/dir2/dir3/file1";
+ String keyName = "dir1/dir2/dir3/file1";
createAndCheck(keyName);
// Key with leading '/'.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org