You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2019/10/13 06:51:04 UTC
[hadoop-ozone] 12/18: Merge remote-tracking branch 'upstream/trunk'
into HDDS-2181
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit a1adf87f4bf73b3421b23d97cdf14a80f58da58a
Merge: 251a4a7 70cf448
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Tue Oct 8 17:55:17 2019 -0700
Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 ++-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 67 ++++-----
.../hdds/scm/client/ContainerOperationClient.java | 10 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 40 +++---
.../hadoop/hdds/scm/storage/CommitWatcher.java | 8 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 6 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 4 +-
.../hdds/ratis/ContainerCommandRequestMessage.java | 107 +++++++++++++++
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 17 ++-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 21 ++-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 12 +-
.../hdds/security/token/BlockTokenVerifier.java | 4 +-
.../security/token/OzoneBlockTokenSelector.java | 8 +-
.../authority/PKIProfiles/DefaultProfile.java | 4 +-
.../hdds/security/x509/keys/HDDSKeyGenerator.java | 6 +-
.../apache/hadoop/hdds/tracing/StringCodec.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 9 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 6 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 20 ++-
.../hadoop/ozone/common/ChecksumByteBuffer.java | 24 ++--
.../helpers/ContainerCommandRequestPBHelper.java | 16 ++-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +-
.../apache/hadoop/ozone/lease/LeaseManager.java | 6 +-
.../ratis/TestContainerCommandRequestMessage.java | 152 +++++++++++++++++++++
.../ozone/container/common/impl/ContainerSet.java | 64 ++++-----
.../container/common/impl/HddsDispatcher.java | 11 +-
.../RandomContainerDeletionChoosingPolicy.java | 11 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 13 +-
.../common/statemachine/EndpointStateMachine.java | 4 +-
.../CloseContainerCommandHandler.java | 6 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 28 ++--
.../server/ratis/ContainerStateMachine.java | 52 ++++---
.../transport/server/ratis/XceiverServerRatis.java | 5 +-
.../container/common/volume/HddsVolumeChecker.java | 14 +-
.../common/volume/ThrottledAsyncChecker.java | 8 +-
.../container/keyvalue/KeyValueBlockIterator.java | 6 +-
.../container/keyvalue/KeyValueContainerCheck.java | 34 ++---
.../ozone/container/keyvalue/KeyValueHandler.java | 84 ++++++++----
.../container/keyvalue/helpers/ChunkUtils.java | 8 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 8 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 9 +-
.../background/BlockDeletingService.java | 6 +-
.../container/ozoneimpl/ContainerController.java | 6 +-
.../container/ozoneimpl/ContainerDataScanner.java | 50 ++++---
.../ozoneimpl/ContainerDataScrubberMetrics.java | 4 +-
.../ozoneimpl/ContainerMetadataScanner.java | 19 ++-
.../ContainerMetadataScrubberMetrics.java | 5 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 17 +++
.../ozone/container/ozoneimpl/OzoneContainer.java | 8 +-
.../container/common/impl/TestContainerSet.java | 18 ++-
.../keyvalue/TestKeyValueContainerCheck.java | 69 ++++------
.../ozoneimpl/TestContainerScrubberMetrics.java | 25 ++--
.../container/ozoneimpl/TestOzoneContainer.java | 23 ++--
.../hadoop/hdds/server/events/EventQueue.java | 2 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 10 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 12 +-
.../scm/command/CommandStatusReportHandler.java | 12 +-
.../container/AbstractContainerReportHandler.java | 6 +-
.../scm/container/ContainerActionsHandler.java | 6 +-
.../hdds/scm/container/ContainerStateManager.java | 4 +-
.../IncrementalContainerReportHandler.java | 6 +-
.../algorithms/SCMContainerPlacementRackAware.java | 6 +-
.../scm/container/states/ContainerAttribute.java | 22 ++-
.../scm/container/states/ContainerStateMap.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 10 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 18 +--
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 118 +++++++++++++---
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
hadoop-ozone/dist/src/main/dockerbin/entrypoint.sh | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 13 ++
.../container/common/TestBlockDeletingService.java | 24 ++--
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 7 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 13 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 38 +++---
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 12 +-
.../om/request/bucket/OMBucketCreateRequest.java | 12 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 16 ++-
.../request/bucket/OMBucketSetPropertyRequest.java | 13 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 6 +-
.../om/request/file/OMDirectoryCreateRequest.java | 4 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 6 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 4 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 4 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 6 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 4 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 22 +--
.../request/s3/bucket/S3BucketDeleteRequest.java | 13 +-
.../S3InitiateMultipartUploadRequest.java | 4 +-
.../multipart/S3MultipartUploadAbortRequest.java | 4 +-
.../S3MultipartUploadCommitPartRequest.java | 4 +-
.../S3MultipartUploadCompleteRequest.java | 4 +-
.../om/request/s3/security/S3GetSecretRequest.java | 6 +-
.../om/request/volume/OMVolumeCreateRequest.java | 10 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 10 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 59 ++++----
106 files changed, 1186 insertions(+), 605 deletions(-)
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index a754f56,20b5174..79500cc
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@@ -178,11 -177,10 +178,11 @@@ public class OMFileCreateRequest extend
OMClientResponse omClientResponse = null;
try {
// check Acl
- checkBucketAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.CREATE);
// acquire lock
- acquiredLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
+ acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
volumeName, bucketName);
OmBucketInfo bucketInfo =
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index c9fdb2d,196d61c..3fe5206
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@@ -140,8 -127,8 +140,8 @@@ public class OMKeyCommitRequest extend
String dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName,
keyName, commitKeyRequest.getClientID());
- bucketLockAcquired = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
- volumeName, bucketName);
- omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, volumeName,
- bucketName);
++ bucketLockAcquired = omMetadataManager.getLock()
++ .acquireWriteLock(BUCKET_LOCK, volumeName, bucketName);
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
omKeyInfo = omMetadataManager.getOpenKeyTable().get(dbOpenKey);
@@@ -179,11 -166,8 +179,11 @@@
ozoneManagerDoubleBufferHelper.add(omClientResponse,
transactionLogIndex));
}
- omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName,
- bucketName);
+
+ if(bucketLockAcquired) {
- omMetadataManager.getLock().releaseLock(BUCKET_LOCK, volumeName,
++ omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName,
+ bucketName);
+ }
}
// Performing audit logging outside of the lock.
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 05e7396,baa13ad..5229e81
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@@ -163,10 -162,9 +163,10 @@@ public class OMKeyCreateRequest extend
OMClientResponse omClientResponse = null;
try {
// check Acl
- checkBucketAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.CREATE);
- acquireLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
+ acquireLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
volumeName, bucketName);
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
//TODO: We can optimize this get here, if getKmsProvider is null, then
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
index 7df1df8,526473c..c594120
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
@@@ -118,14 -117,10 +118,14 @@@ public class OMKeyRenameRequest extend
throw new OMException("Key name is empty",
OMException.ResultCodes.INVALID_KEY_NAME);
}
- // check Acl
- checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName);
+ // check Acls to see if user has access to perform delete operation on
+ // old key and create operation on new key
+ checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName,
+ IAccessAuthorizer.ACLType.DELETE);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, toKeyName,
+ IAccessAuthorizer.ACLType.CREATE);
- acquiredLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
+ acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
volumeName, bucketName);
// Not doing bucket/volume checks here. In this way we can avoid db
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org