You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2019/10/13 06:51:09 UTC
[hadoop-ozone] 17/18: Merge remote-tracking branch 'upstream/trunk'
into HDDS-2181
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 6828f2e3aa7e8a937544e0b70ec844062376f996
Merge: ac4990f 640255a
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Oct 10 19:23:16 2019 -0700
Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
.../hadoop/hdds/scm/XceiverClientManager.java | 7 +
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 3 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 15 +
.../hadoop/hdds/scm/ByteStringConversion.java | 62 +++
.../apache/hadoop/hdds/scm/ByteStringHelper.java | 69 ----
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 3 +-
.../hadoop/hdds/utils/db/cache/CacheKey.java | 11 +-
.../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 12 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 3 +
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 11 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 19 +-
.../hadoop/ozone/lock/PooledLockFactory.java | 7 +-
.../common/src/main/resources/ozone-default.xml | 11 +
.../ozone/container/keyvalue/KeyValueHandler.java | 33 +-
.../container/keyvalue/helpers/ChunkUtils.java | 34 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 6 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 60 ++-
.../keyvalue/interfaces/ChunkManager.java | 2 +-
.../container/keyvalue/TestChunkManagerImpl.java | 69 ++--
.../client/io/BlockOutputStreamEntryPool.java | 22 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 6 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 +-
.../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 6 +-
.../hadoop/ozone/om/helpers/OMRatisHelper.java | 4 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 31 +-
.../security/OzoneBlockTokenSecretManager.java | 2 +-
.../OzoneDelegationTokenSecretManager.java | 6 +-
.../security/OzoneDelegationTokenSelector.java | 8 +-
.../hadoop/ozone/security/OzoneSecretManager.java | 6 +-
.../dev-support/checks/_mvn_unit_report.sh | 5 +
.../dist/src/main/compose/ozone-hdfs/docker-config | 46 ---
.../dist/src/main/compose/ozone-mr/common-config | 9 -
.../src/main/compose/ozone-om-ha/docker-config | 45 ---
.../src/main/compose/ozone-recon/docker-config | 47 +--
.../src/main/compose/ozone-topology/docker-config | 49 ---
.../dist/src/main/compose/ozone/docker-config | 45 ---
.../src/main/compose/ozoneblockade/docker-config | 45 ---
.../dist/src/main/compose/ozoneperf/docker-config | 13 -
.../src/main/compose/ozones3-haproxy/docker-config | 48 ---
.../dist/src/main/compose/ozones3/docker-config | 48 ---
.../src/main/compose/ozonescripts/docker-config | 7 +-
.../src/main/compose/ozonesecure-mr/docker-config | 46 ---
.../src/main/compose/ozonesecure/docker-config | 53 ---
.../ozone/container/ContainerTestHelper.java | 11 +-
.../common/impl/TestContainerPersistence.java | 53 +--
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 6 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 27 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 125 ++++--
.../hadoop/ozone/om/OpenKeyCleanupService.java | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 11 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 16 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 8 +-
.../ozone/om/ratis/OzoneManagerRatisClient.java | 53 +--
.../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 4 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 6 +-
.../OzoneManagerHARequestHandlerImpl.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 4 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 417 +++++++++++++++++++++
.../ozone/om/request/TestOMRequestUtils.java | 60 ++-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../apache/hadoop/ozone/s3/AWSV4AuthParser.java | 10 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 5 +-
.../ozone/s3/exception/OS3ExceptionMapper.java | 4 +-
68 files changed, 1040 insertions(+), 873 deletions(-)
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 19976e5,20b7fdf..b451722
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@@ -1670,11 -1674,8 +1670,13 @@@ public class KeyManagerImpl implements
}
if (keyInfo == null) {
- throw new OMException("Key not found, checkAccess failed. Key:" +
- objectKey, KEY_NOT_FOUND);
+ // the key does not exist, but it is a parent "dir" of some key
+ // let access be determined based on volume/bucket/prefix ACL
- LOG.debug("key:{} is non-existent parent, permit access to user:{}",
- keyName, context.getClientUgi());
++ if (LOG.isDebugEnabled()) {
++ LOG.debug("key:{} is non-existent parent, permit access to user:{}",
++ keyName, context.getClientUgi());
++ }
+ return true;
}
boolean hasAccess = OzoneAclUtil.checkAclRight(
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
index d974537,0b7c51a..442dc59
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
@@@ -80,53 -77,25 +80,53 @@@ public class OzoneNativeAuthorizer impl
"configured to work with OzoneObjInfo type only.", INVALID_REQUEST);
}
+ // For CREATE and DELETE acl requests, the parents need to be checked
+ // for WRITE acl. If Key create request is received, then we need to
+ // check if user has WRITE acl set on Bucket and Volume. In all other cases
+ // the parents also need to be checked for the same acl type.
+ if (isACLTypeCreate || isACLTypeDelete) {
+ parentContext = RequestContext.newBuilder()
+ .setClientUgi(context.getClientUgi())
+ .setIp(context.getIp())
+ .setAclType(context.getAclType())
+ .setAclRights(ACLType.WRITE)
+ .build();
+ } else {
+ parentContext = context;
+ }
+
switch (objInfo.getResourceType()) {
case VOLUME:
- LOG.trace("Checking access for volume:" + objInfo);
+ LOG.trace("Checking access for volume: {}", objInfo);
return volumeManager.checkAccess(objInfo, context);
case BUCKET:
- LOG.trace("Checking access for bucket:" + objInfo);
+ LOG.trace("Checking access for bucket: {}", objInfo);
- return (bucketManager.checkAccess(objInfo, context)
- && volumeManager.checkAccess(objInfo, context));
+ // Skip bucket access check for CREATE acl since
+ // bucket will not exist at the time of creation
+ boolean bucketAccess = isACLTypeCreate
+ || bucketManager.checkAccess(objInfo, context);
+ return (bucketAccess
+ && volumeManager.checkAccess(objInfo, parentContext));
case KEY:
+ case OPEN_KEY:
- LOG.trace("Checking access for Key:" + objInfo);
+ LOG.trace("Checking access for Key: {}", objInfo);
- return (keyManager.checkAccess(objInfo, context)
- && prefixManager.checkAccess(objInfo, context)
- && bucketManager.checkAccess(objInfo, context)
- && volumeManager.checkAccess(objInfo, context));
+ // Skip key access check for CREATE acl since
+ // key will not exist at the time of creation
+ boolean keyAccess = isACLTypeCreate
+ || keyManager.checkAccess(objInfo, context);
+ return (keyAccess
+ && prefixManager.checkAccess(objInfo, parentContext)
+ && bucketManager.checkAccess(objInfo, parentContext)
+ && volumeManager.checkAccess(objInfo, parentContext));
case PREFIX:
- LOG.trace("Checking access for Prefix:" + objInfo);
- LOG.trace("Checking access for Prefix: {]", objInfo);
- return (prefixManager.checkAccess(objInfo, context)
- && bucketManager.checkAccess(objInfo, context)
- && volumeManager.checkAccess(objInfo, context));
++ LOG.trace("Checking access for Prefix: {}", objInfo);
+ // Skip prefix access check for CREATE acl since
+ // prefix will not exist at the time of creation
+ boolean prefixAccess = isACLTypeCreate
+ || prefixManager.checkAccess(objInfo, context);
+ return (prefixAccess
+ && bucketManager.checkAccess(objInfo, parentContext)
+ && volumeManager.checkAccess(objInfo, parentContext));
default:
throw new OMException("Unexpected object type:" +
objInfo.getResourceType(), INVALID_REQUEST);
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org