You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2019/10/13 06:50:52 UTC
[hadoop-ozone] branch HDDS-2181 created (now b141674)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a change to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
at b141674 Fix review comments
This branch includes the following new commits:
new fe572e6 HDDS-2181. Ozone Manager should send correct ACL type in ACL requests to Authorizer
new 436f493 Fix review comments
new a837ada Add delete acl to key rename request
new cfabf1a Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
new 4fad1bd Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
new ac66e63 Fix acceptance tests and native authorizer
new 8359a8a Fix review comments
new 5c5e887 Fix checkstyle issues
new d41b7b9 Fix integration test failures
new 00a1160 Fix unit test failures
new 251a4a7 Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
new a1adf87 Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
new f926998 Handle acl checks correctly in allocate block request
new 076d05c Fix unit test failures
new 90ca124 Fix review comments
new ac4990f Fix acceptance test failures
new 6828f2e Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
new b141674 Fix review comments
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 07/18: Fix review comments
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 8359a8a6026c06a9cf5f0f787a4055752dc5135b
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Oct 3 17:52:09 2019 -0700
Fix review comments
---
.../java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index bf4148d..854048b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -123,7 +123,7 @@ public class OMKeyCommitRequest extends OMKeyRequest {
// Native authorizer requires client id as part of keyname to check
// write ACL on key. Add client id to key name if ozone native
// authorizer is configured.
- Configuration config = new OzoneConfiguration();
+ Configuration config = ozoneManager.getConfiguration();
String keyNameForAclCheck = keyName;
if (OmUtils.isNativeAuthorizerEnabled(config)) {
keyNameForAclCheck = keyName + "/" + commitKeyRequest.getClientID();
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 08/18: Fix checkstyle issues
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 5c5e887f088c249c08c27a567592217942a60691
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Oct 3 18:47:42 2019 -0700
Fix checkstyle issues
---
.../java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 854048b..c9fdb2d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -26,7 +26,6 @@ import java.util.stream.Collectors;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 06/18: Fix acceptance tests and native authorizer
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit ac66e6348c8d911b3b0731bd84e8b868b600fbe2
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Oct 3 15:37:54 2019 -0700
Fix acceptance tests and native authorizer
---
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 14 +++++++
.../ozone/om/request/key/OMKeyCommitRequest.java | 25 ++++++++---
.../ozone/security/acl/OzoneNativeAuthorizer.java | 48 ++++++++++++++++++----
3 files changed, 73 insertions(+), 14 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 7cd38ad..b5ce46b 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -52,6 +52,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BIND_HOST_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY;
@@ -530,4 +532,16 @@ public final class OmUtils {
return repeatedOmKeyInfo;
}
+
+ /**
+ * Returns true if OzoneNativeAuthorizer is configured in the configuration.
+ * @param configuration ozone configuration
+ * @return true if OzoneNativeAuthorizer is configured in the configuration;
+ * else false.
+ */
+ public static boolean isNativeAuthorizerEnabled(Configuration configuration) {
+ String authorizer = configuration.get(OZONE_ACL_AUTHORIZER_CLASS);
+ return authorizer != null &&
+ authorizer.equals(OZONE_ACL_AUTHORIZER_CLASS_NATIVE);
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index b349fa9..bf4148d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -25,6 +25,9 @@ import java.util.stream.Collectors;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.slf4j.Logger;
@@ -112,11 +115,20 @@ public class OMKeyCommitRequest extends OMKeyRequest {
IOException exception = null;
OmKeyInfo omKeyInfo = null;
OMClientResponse omClientResponse = null;
+ boolean bucketLockAcquired = false;
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
try {
// check Acl
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ // Native authorizer requires client id as part of keyname to check
+ // write ACL on key. Add client id to key name if ozone native
+ // authorizer is configured.
+ Configuration config = new OzoneConfiguration();
+ String keyNameForAclCheck = keyName;
+ if (OmUtils.isNativeAuthorizerEnabled(config)) {
+ keyNameForAclCheck = keyName + "/" + commitKeyRequest.getClientID();
+ }
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyNameForAclCheck,
IAccessAuthorizer.ACLType.WRITE);
List<OmKeyLocationInfo> locationInfoList = commitKeyArgs
@@ -129,8 +141,8 @@ public class OMKeyCommitRequest extends OMKeyRequest {
String dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName,
keyName, commitKeyRequest.getClientID());
- omMetadataManager.getLock().acquireLock(BUCKET_LOCK, volumeName,
- bucketName);
+ bucketLockAcquired = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
+ volumeName, bucketName);
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
omKeyInfo = omMetadataManager.getOpenKeyTable().get(dbOpenKey);
@@ -168,8 +180,11 @@ public class OMKeyCommitRequest extends OMKeyRequest {
ozoneManagerDoubleBufferHelper.add(omClientResponse,
transactionLogIndex));
}
- omMetadataManager.getLock().releaseLock(BUCKET_LOCK, volumeName,
- bucketName);
+
+ if(bucketLockAcquired) {
+ omMetadataManager.getLock().releaseLock(BUCKET_LOCK, volumeName,
+ bucketName);
+ }
}
// Performing audit logging outside of the lock.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
index 5acd37e..1731421 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
@@ -69,6 +69,9 @@ public class OzoneNativeAuthorizer implements IAccessAuthorizer {
Objects.requireNonNull(ozObject);
Objects.requireNonNull(context);
OzoneObjInfo objInfo;
+ RequestContext parentContext;
+ boolean isACLTypeCreate = (context.getAclRights() == ACLType.CREATE);
+ boolean isACLTypeDelete = (context.getAclRights() == ACLType.DELETE);
if (ozObject instanceof OzoneObjInfo) {
objInfo = (OzoneObjInfo) ozObject;
@@ -77,25 +80,52 @@ public class OzoneNativeAuthorizer implements IAccessAuthorizer {
"configured to work with OzoneObjInfo type only.", INVALID_REQUEST);
}
+ // For CREATE and DELETE acl requests, the parents need to be checked
+ // for WRITE acl. If Key create request is received, then we need to
+ // check if user has WRITE acl set on Bucket and Volume. In all other cases
+ // the parents also need to be checked for the same acl type.
+ if (isACLTypeCreate || isACLTypeDelete) {
+ parentContext = RequestContext.newBuilder()
+ .setClientUgi(context.getClientUgi())
+ .setIp(context.getIp())
+ .setAclType(context.getAclType())
+ .setAclRights(ACLType.WRITE)
+ .build();
+ } else {
+ parentContext = context;
+ }
+
switch (objInfo.getResourceType()) {
case VOLUME:
LOG.trace("Checking access for volume:" + objInfo);
return volumeManager.checkAccess(objInfo, context);
case BUCKET:
LOG.trace("Checking access for bucket:" + objInfo);
- return (bucketManager.checkAccess(objInfo, context)
- && volumeManager.checkAccess(objInfo, context));
+ // Skip bucket access check for CREATE acl since
+ // bucket will not exist at the time of creation
+ boolean bucketAccess = isACLTypeCreate
+ || bucketManager.checkAccess(objInfo, context);
+ return (bucketAccess
+ && volumeManager.checkAccess(objInfo, parentContext));
case KEY:
LOG.trace("Checking access for Key:" + objInfo);
- return (keyManager.checkAccess(objInfo, context)
- && prefixManager.checkAccess(objInfo, context)
- && bucketManager.checkAccess(objInfo, context)
- && volumeManager.checkAccess(objInfo, context));
+ // Skip key access check for CREATE acl since
+ // key will not exist at the time of creation
+ boolean keyAccess = isACLTypeCreate
+ || keyManager.checkAccess(objInfo, context);
+ return (keyAccess
+ && prefixManager.checkAccess(objInfo, parentContext)
+ && bucketManager.checkAccess(objInfo, parentContext)
+ && volumeManager.checkAccess(objInfo, parentContext));
case PREFIX:
LOG.trace("Checking access for Prefix:" + objInfo);
- return (prefixManager.checkAccess(objInfo, context)
- && bucketManager.checkAccess(objInfo, context)
- && volumeManager.checkAccess(objInfo, context));
+ // Skip prefix access check for CREATE acl since
+ // prefix will not exist at the time of creation
+ boolean prefixAccess = isACLTypeCreate
+ || prefixManager.checkAccess(objInfo, context);
+ return (prefixAccess
+ && bucketManager.checkAccess(objInfo, parentContext)
+ && volumeManager.checkAccess(objInfo, parentContext));
default:
throw new OMException("Unexpected object type:" +
objInfo.getResourceType(), INVALID_REQUEST);
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 15/18: Fix review comments
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 90ca1247b36fa384ae17ddd2dd11a7a5603ae72d
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Wed Oct 9 17:11:55 2019 -0700
Fix review comments
---
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 14 ----------
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 32 ++++++++--------------
.../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +++++++
.../om/request/key/OMAllocateBlockRequest.java | 21 ++------------
.../ozone/om/request/key/OMKeyCommitRequest.java | 22 ++-------------
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 31 +++++++++++++++++++++
6 files changed, 57 insertions(+), 73 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index ad33cae..8e129c9 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -57,8 +57,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BIND_HOST_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY;
@@ -527,16 +525,4 @@ public final class OmUtils {
return repeatedOmKeyInfo;
}
-
- /**
- * Returns true if OzoneNativeAuthorizer is configured in the configuration.
- * @param configuration ozone configuration
- * @return true if OzoneNativeAuthorizer is configured in the configuration;
- * else false.
- */
- public static boolean isNativeAuthorizerEnabled(Configuration configuration) {
- String authorizer = configuration.get(OZONE_ACL_AUTHORIZER_CLASS);
- return authorizer != null &&
- authorizer.equals(OZONE_ACL_AUTHORIZER_CLASS_NATIVE);
- }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index faa65bb..d0be40b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -1655,31 +1655,21 @@ public class KeyManagerImpl implements KeyManager {
metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volume, bucket);
try {
validateBucket(volume, bucket);
- OmKeyInfo keyInfo = null;
- try {
- if (ozObject.getResourceType() == OPEN_KEY) {
- keyInfo = metadataManager.getOpenKeyTable().get(objectKey);
- } else {
- OzoneFileStatus fileStatus = getFileStatus(args);
- keyInfo = fileStatus.getKeyInfo();
- }
+ OmKeyInfo keyInfo;
- if (keyInfo == null) {
- // the key does not exist, but it is a parent "dir" of some key
- // let access be determined based on volume/bucket/prefix ACL
- LOG.debug("key:{} is non-existent parent, permit access to user:{}",
- keyName, context.getClientUgi());
- return true;
- }
- } catch (OMException e) {
- if (e.getResult() == FILE_NOT_FOUND) {
- keyInfo = metadataManager.getOpenKeyTable().get(objectKey);
- }
+ if (ozObject.getResourceType() == OPEN_KEY) {
+ keyInfo = metadataManager.getOpenKeyTable().get(objectKey);
+ } else {
+ OzoneFileStatus fileStatus = getFileStatus(args);
+ keyInfo = fileStatus.getKeyInfo();
}
if (keyInfo == null) {
- throw new OMException("Key not found, checkAccess failed. Key:" +
- objectKey, KEY_NOT_FOUND);
+ // the key does not exist, but it is a parent "dir" of some key
+ // let access be determined based on volume/bucket/prefix ACL
+ LOG.debug("key:{} is non-existent parent, permit access to user:{}",
+ keyName, context.getClientUgi());
+ return true;
}
boolean hasAccess = OzoneAclUtil.checkAclRight(
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a6503d7..ba157bc 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -301,6 +301,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private final boolean grpcBlockTokenEnabled;
private final boolean useRatisForReplication;
+ private boolean isNativeAuthorizerEnabled;
+
private OzoneManager(OzoneConfiguration conf) throws IOException,
AuthenticationException {
super(OzoneVersionInfo.OZONE_VERSION_INFO);
@@ -473,6 +475,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
if (accessAuthorizer instanceof OzoneNativeAuthorizer) {
OzoneNativeAuthorizer authorizer =
(OzoneNativeAuthorizer) accessAuthorizer;
+ isNativeAuthorizerEnabled = true;
authorizer.setVolumeManager(volumeManager);
authorizer.setBucketManager(bucketManager);
authorizer.setKeyManager(keyManager);
@@ -3290,4 +3293,11 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
return ozAdmins;
}
+ /**
+ * Returns true if OzoneNativeAuthorizer is enabled and false if otherwise.
+ * @return if native authorizer is enabled.
+ */
+ public boolean isNativeAuthorizerEnabled() {
+ return isNativeAuthorizerEnabled;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index ef2af6d..7bc8738 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -25,11 +25,8 @@ import java.util.Map;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -173,22 +170,8 @@ public class OMAllocateBlockRequest extends OMKeyRequest {
OmKeyInfo omKeyInfo = null;
try {
// check Acl
- // Native authorizer requires client id as part of keyname to check
- // write ACL on key. Add client id to key name if ozone native
- // authorizer is configured.
- Configuration config = ozoneManager.getConfiguration();
- if (OmUtils.isNativeAuthorizerEnabled(config)) {
- String keyNameForAclCheck =
- keyName + "/" + allocateBlockRequest.getClientID();
- // During allocate block request, it is possible that key is
- // not present in the key table and hence setting the resource type
- // to OPEN_KEY to check the openKeyTable.
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyNameForAclCheck,
- IAccessAuthorizer.ACLType.WRITE, OzoneObj.ResourceType.OPEN_KEY);
- } else {
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
- IAccessAuthorizer.ACLType.WRITE, OzoneObj.ResourceType.KEY);
- }
+ checkKeyAclsInOpenKeyTable(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.WRITE, allocateBlockRequest.getClientID());
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
validateBucketAndVolume(omMetadataManager, volumeName,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 63ea5a0..811ecf7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -25,11 +25,8 @@ import java.util.stream.Collectors;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
-import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -120,22 +117,9 @@ public class OMKeyCommitRequest extends OMKeyRequest {
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
try {
// check Acl
- // Native authorizer requires client id as part of keyname to check
- // write ACL on key. Add client id to key name if ozone native
- // authorizer is configured.
- Configuration config = ozoneManager.getConfiguration();
- if (OmUtils.isNativeAuthorizerEnabled(config)) {
- String keyNameForAclCheck =
- keyName + "/" + commitKeyRequest.getClientID();
- // During key commit request, it is possible that key is
- // not present in the key table and hence setting the resource type
- // to OPEN_KEY to check the openKeyTable.
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyNameForAclCheck,
- IAccessAuthorizer.ACLType.WRITE, OzoneObj.ResourceType.OPEN_KEY);
- } else {
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
- IAccessAuthorizer.ACLType.WRITE, OzoneObj.ResourceType.KEY);
- }
+ checkKeyAclsInOpenKeyTable(ozoneManager, volumeName, bucketName,
+ keyName, IAccessAuthorizer.ACLType.WRITE,
+ commitKeyRequest.getClientID());
List<OmKeyLocationInfo> locationInfoList = commitKeyArgs
.getKeyLocationsList().stream()
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 16e97e8..73753d8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -523,6 +523,8 @@ public abstract class OMKeyRequest extends OMClientRequest {
* @param volume
* @param bucket
* @param key
+ * @param aclType
+ * @param resourceType
* @throws IOException
*/
protected void checkKeyAcls(OzoneManager ozoneManager, String volume,
@@ -535,4 +537,33 @@ public abstract class OMKeyRequest extends OMClientRequest {
}
}
+ /**
+ * Check ACLs for Ozone Key in OpenKey table
+ * if ozone native authorizer is enabled.
+ * @param ozoneManager
+ * @param volume
+ * @param bucket
+ * @param key
+ * @param aclType
+ * @param clientId
+ * @throws IOException
+ */
+ protected void checkKeyAclsInOpenKeyTable(OzoneManager ozoneManager,
+ String volume, String bucket, String key,
+ IAccessAuthorizer.ACLType aclType, long clientId) throws IOException {
+ // Native authorizer requires client id as part of key name to check
+ // write ACL on key. Add client id to key name if ozone native
+ // authorizer is configured.
+ if (ozoneManager.isNativeAuthorizerEnabled()) {
+ String keyNameForAclCheck = key + "/" + clientId;
+ // During key commit request, it is possible that key is
+ // not present in the key table and hence setting the resource type
+ // to OPEN_KEY to check the openKeyTable.
+ checkKeyAcls(ozoneManager, volume, bucket, keyNameForAclCheck,
+ aclType, OzoneObj.ResourceType.OPEN_KEY);
+ } else {
+ checkKeyAcls(ozoneManager, volume, bucket, key,
+ aclType, OzoneObj.ResourceType.KEY);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 09/18: Fix integration test failures
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit d41b7b9a71645080e420de391e724e916dcf6250
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Fri Oct 4 13:28:11 2019 -0700
Fix integration test failures
---
.../src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java
index c75e365..ebf5964 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java
@@ -118,7 +118,7 @@ public class TestOmAcls {
() -> volume.createBucket(bucketName));
assertTrue(logCapturer.getOutput()
- .contains("doesn't have CREATE permission to access volume"));
+ .contains("doesn't have CREATE permission to access bucket"));
}
@Test
@@ -133,8 +133,8 @@ public class TestOmAcls {
OzoneTestUtils.expectOmException(ResultCodes.PERMISSION_DENIED,
() -> TestDataUtil.createKey(bucket, "testKey", "testcontent"));
- assertTrue(logCapturer.getOutput().contains("doesn't have WRITE " +
- "permission to access bucket"));
+ assertTrue(logCapturer.getOutput().contains("doesn't have CREATE " +
+ "permission to access key"));
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 17/18: Merge remote-tracking branch 'upstream/trunk'
into HDDS-2181
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 6828f2e3aa7e8a937544e0b70ec844062376f996
Merge: ac4990f 640255a
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Oct 10 19:23:16 2019 -0700
Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
.../hadoop/hdds/scm/XceiverClientManager.java | 7 +
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 3 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 15 +
.../hadoop/hdds/scm/ByteStringConversion.java | 62 +++
.../apache/hadoop/hdds/scm/ByteStringHelper.java | 69 ----
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 3 +-
.../hadoop/hdds/utils/db/cache/CacheKey.java | 11 +-
.../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 12 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 3 +
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 11 +-
.../org/apache/hadoop/ozone/lock/LockManager.java | 19 +-
.../hadoop/ozone/lock/PooledLockFactory.java | 7 +-
.../common/src/main/resources/ozone-default.xml | 11 +
.../ozone/container/keyvalue/KeyValueHandler.java | 33 +-
.../container/keyvalue/helpers/ChunkUtils.java | 34 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 6 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 60 ++-
.../keyvalue/interfaces/ChunkManager.java | 2 +-
.../container/keyvalue/TestChunkManagerImpl.java | 69 ++--
.../client/io/BlockOutputStreamEntryPool.java | 22 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 6 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 +-
.../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 6 +-
.../hadoop/ozone/om/helpers/OMRatisHelper.java | 4 +-
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 31 +-
.../security/OzoneBlockTokenSecretManager.java | 2 +-
.../OzoneDelegationTokenSecretManager.java | 6 +-
.../security/OzoneDelegationTokenSelector.java | 8 +-
.../hadoop/ozone/security/OzoneSecretManager.java | 6 +-
.../dev-support/checks/_mvn_unit_report.sh | 5 +
.../dist/src/main/compose/ozone-hdfs/docker-config | 46 ---
.../dist/src/main/compose/ozone-mr/common-config | 9 -
.../src/main/compose/ozone-om-ha/docker-config | 45 ---
.../src/main/compose/ozone-recon/docker-config | 47 +--
.../src/main/compose/ozone-topology/docker-config | 49 ---
.../dist/src/main/compose/ozone/docker-config | 45 ---
.../src/main/compose/ozoneblockade/docker-config | 45 ---
.../dist/src/main/compose/ozoneperf/docker-config | 13 -
.../src/main/compose/ozones3-haproxy/docker-config | 48 ---
.../dist/src/main/compose/ozones3/docker-config | 48 ---
.../src/main/compose/ozonescripts/docker-config | 7 +-
.../src/main/compose/ozonesecure-mr/docker-config | 46 ---
.../src/main/compose/ozonesecure/docker-config | 53 ---
.../ozone/container/ContainerTestHelper.java | 11 +-
.../common/impl/TestContainerPersistence.java | 53 +--
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 6 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 27 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 125 ++++--
.../hadoop/ozone/om/OpenKeyCleanupService.java | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 11 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 16 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 8 +-
.../ozone/om/ratis/OzoneManagerRatisClient.java | 53 +--
.../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 4 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 6 +-
.../OzoneManagerHARequestHandlerImpl.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 4 +-
.../protocolPB/OzoneManagerRequestHandler.java | 4 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 8 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 417 +++++++++++++++++++++
.../ozone/om/request/TestOMRequestUtils.java | 60 ++-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../apache/hadoop/ozone/s3/AWSV4AuthParser.java | 10 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 5 +-
.../ozone/s3/exception/OS3ExceptionMapper.java | 4 +-
68 files changed, 1040 insertions(+), 873 deletions(-)
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 19976e5,20b7fdf..b451722
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@@ -1670,11 -1674,8 +1670,13 @@@ public class KeyManagerImpl implements
}
if (keyInfo == null) {
- throw new OMException("Key not found, checkAccess failed. Key:" +
- objectKey, KEY_NOT_FOUND);
+ // the key does not exist, but it is a parent "dir" of some key
+ // let access be determined based on volume/bucket/prefix ACL
- LOG.debug("key:{} is non-existent parent, permit access to user:{}",
- keyName, context.getClientUgi());
++ if (LOG.isDebugEnabled()) {
++ LOG.debug("key:{} is non-existent parent, permit access to user:{}",
++ keyName, context.getClientUgi());
++ }
+ return true;
}
boolean hasAccess = OzoneAclUtil.checkAclRight(
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
index d974537,0b7c51a..442dc59
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
@@@ -80,53 -77,25 +80,53 @@@ public class OzoneNativeAuthorizer impl
"configured to work with OzoneObjInfo type only.", INVALID_REQUEST);
}
+ // For CREATE and DELETE acl requests, the parents need to be checked
+ // for WRITE acl. If Key create request is received, then we need to
+ // check if user has WRITE acl set on Bucket and Volume. In all other cases
+ // the parents also need to be checked for the same acl type.
+ if (isACLTypeCreate || isACLTypeDelete) {
+ parentContext = RequestContext.newBuilder()
+ .setClientUgi(context.getClientUgi())
+ .setIp(context.getIp())
+ .setAclType(context.getAclType())
+ .setAclRights(ACLType.WRITE)
+ .build();
+ } else {
+ parentContext = context;
+ }
+
switch (objInfo.getResourceType()) {
case VOLUME:
- LOG.trace("Checking access for volume:" + objInfo);
+ LOG.trace("Checking access for volume: {}", objInfo);
return volumeManager.checkAccess(objInfo, context);
case BUCKET:
- LOG.trace("Checking access for bucket:" + objInfo);
+ LOG.trace("Checking access for bucket: {}", objInfo);
- return (bucketManager.checkAccess(objInfo, context)
- && volumeManager.checkAccess(objInfo, context));
+ // Skip bucket access check for CREATE acl since
+ // bucket will not exist at the time of creation
+ boolean bucketAccess = isACLTypeCreate
+ || bucketManager.checkAccess(objInfo, context);
+ return (bucketAccess
+ && volumeManager.checkAccess(objInfo, parentContext));
case KEY:
+ case OPEN_KEY:
- LOG.trace("Checking access for Key:" + objInfo);
+ LOG.trace("Checking access for Key: {}", objInfo);
- return (keyManager.checkAccess(objInfo, context)
- && prefixManager.checkAccess(objInfo, context)
- && bucketManager.checkAccess(objInfo, context)
- && volumeManager.checkAccess(objInfo, context));
+ // Skip key access check for CREATE acl since
+ // key will not exist at the time of creation
+ boolean keyAccess = isACLTypeCreate
+ || keyManager.checkAccess(objInfo, context);
+ return (keyAccess
+ && prefixManager.checkAccess(objInfo, parentContext)
+ && bucketManager.checkAccess(objInfo, parentContext)
+ && volumeManager.checkAccess(objInfo, parentContext));
case PREFIX:
- LOG.trace("Checking access for Prefix:" + objInfo);
- LOG.trace("Checking access for Prefix: {]", objInfo);
- return (prefixManager.checkAccess(objInfo, context)
- && bucketManager.checkAccess(objInfo, context)
- && volumeManager.checkAccess(objInfo, context));
++ LOG.trace("Checking access for Prefix: {}", objInfo);
+ // Skip prefix access check for CREATE acl since
+ // prefix will not exist at the time of creation
+ boolean prefixAccess = isACLTypeCreate
+ || prefixManager.checkAccess(objInfo, context);
+ return (prefixAccess
+ && bucketManager.checkAccess(objInfo, parentContext)
+ && volumeManager.checkAccess(objInfo, parentContext));
default:
throw new OMException("Unexpected object type:" +
objInfo.getResourceType(), INVALID_REQUEST);
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 18/18: Fix review comments
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit b141674ee5d203e0e66f7ef1a20d232cdc6f7eb6
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Oct 10 22:55:32 2019 -0700
Fix review comments
---
.../src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index b451722..0b23b63 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -1663,7 +1663,7 @@ public class KeyManagerImpl implements KeyManager {
try {
OzoneFileStatus fileStatus = getFileStatus(args);
keyInfo = fileStatus.getKeyInfo();
- } catch (Exception e) {
+ } catch (IOException e) {
throw new OMException("Key not found, checkAccess failed. Key:" +
objectKey, KEY_NOT_FOUND);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 02/18: Fix review comments
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 436f493d7d59ecba6a340e4f0486feec6bba6226
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Sep 26 12:12:16 2019 -0700
Fix review comments
---
.../apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java | 2 +-
.../java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
index 568c939..632f173 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java
@@ -93,7 +93,7 @@ public class OMBucketDeleteRequest extends OMClientRequest {
// check Acl
if (ozoneManager.getAclsEnabled()) {
checkAcls(ozoneManager, OzoneObj.ResourceType.BUCKET,
- OzoneObj.StoreType.OZONE, IAccessAuthorizer.ACLType.WRITE,
+ OzoneObj.StoreType.OZONE, IAccessAuthorizer.ACLType.DELETE,
volumeName, bucketName, null);
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 622deb8..b349fa9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -117,7 +117,7 @@ public class OMKeyCommitRequest extends OMKeyRequest {
try {
// check Acl
checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
- IAccessAuthorizer.ACLType.CREATE);
+ IAccessAuthorizer.ACLType.WRITE);
List<OmKeyLocationInfo> locationInfoList = commitKeyArgs
.getKeyLocationsList().stream()
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 11/18: Merge remote-tracking branch 'upstream/trunk'
into HDDS-2181
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 251a4a72d39d8bb0e7d7571e2a5148b4a8e503cf
Merge: 00a1160 5583014
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Fri Oct 4 15:05:38 2019 -0700
Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
.../common/dev-support/findbugsExcludeFile.xml | 5 +
.../hadoop/hdds/scm/container/ContainerInfo.java | 2 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 24 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 2 +-
.../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 3 +-
.../hadoop/hdds/utils/db/RocksDBConfiguration.java | 62 +++
.../hadoop/ozone/common/ChecksumByteBuffer.java | 114 +++++
.../ozone/common/PureJavaCrc32ByteBuffer.java | 556 ++++++++++++++++++++
.../ozone/common/PureJavaCrc32CByteBuffer.java | 559 +++++++++++++++++++++
.../org/apache/hadoop/ozone/lock/ActiveLock.java | 63 ++-
.../org/apache/hadoop/ozone/lock/LockManager.java | 166 +++++-
.../apache/hadoop/ozone/web/utils/JsonUtils.java | 5 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 16 +-
.../ozone/common/TestChecksumByteBuffer.java | 102 ++++
.../apache/hadoop/ozone/lock/TestLockManager.java | 145 +++++-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 8 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 51 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 26 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 7 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 36 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 67 ++-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 5 +-
.../hdds/scm/cli/container/ListSubcommand.java | 4 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 97 ++--
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 79 ++-
hadoop-ozone/dev-support/checks/blockade.sh | 2 +-
hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env | 2 +-
.../main/compose/ozone-hdfs/docker-compose.yaml | 6 +-
.../dist/src/main/compose/ozone-mr/hadoop27/.env | 2 +-
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 2 +-
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop32/.env | 2 +-
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-om-ha/.env | 2 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 10 +-
.../dist/src/main/compose/ozone-recon/.env | 2 +-
.../main/compose/ozone-recon/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-topology/.env | 2 +-
.../compose/ozone-topology/docker-compose.yaml | 12 +-
hadoop-ozone/dist/src/main/compose/ozone/.env | 2 +-
.../src/main/compose/ozone/docker-compose.yaml | 6 +-
.../dist/src/main/compose/ozoneblockade/.env | 2 +-
.../main/compose/ozoneblockade/docker-compose.yaml | 8 +-
hadoop-ozone/dist/src/main/compose/ozoneperf/.env | 2 +-
.../src/main/compose/ozoneperf/docker-compose.yaml | 10 +-
.../dist/src/main/compose/ozones3-haproxy/.env | 2 +-
.../compose/ozones3-haproxy/docker-compose.yaml | 12 +-
hadoop-ozone/dist/src/main/compose/ozones3/.env | 2 +-
.../src/main/compose/ozones3/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozonescripts/.env | 2 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../compose/ozonesecure-mr/docker-compose.yaml | 50 +-
.../src/main/compose/ozonesecure-mr/docker-config | 28 +-
.../dist/src/main/compose/ozonesecure/.env | 2 +-
.../main/compose/ozonesecure/docker-compose.yaml | 10 +-
.../dist/src/main/smoketest/gdpr/gdpr.robot | 89 ++++
.../src/test/blockade/ozone/cluster.py | 4 +-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 4 -
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 97 ++++
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 1 -
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 59 +--
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 10 -
.../hadoop/ozone/web/ozShell/ObjectPrinter.java | 3 +-
.../web/ozShell/bucket/AddAclBucketHandler.java | 5 +-
.../web/ozShell/bucket/GetAclBucketHandler.java | 4 +-
.../web/ozShell/bucket/RemoveAclBucketHandler.java | 7 +-
.../web/ozShell/bucket/SetAclBucketHandler.java | 5 +-
.../ozone/web/ozShell/keys/AddAclKeyHandler.java | 5 +-
.../ozone/web/ozShell/keys/GetAclKeyHandler.java | 4 +-
.../web/ozShell/keys/RemoveAclKeyHandler.java | 7 +-
.../ozone/web/ozShell/keys/SetAclKeyHandler.java | 5 +-
.../ozone/web/ozShell/token/GetTokenHandler.java | 2 +-
.../ozone/web/ozShell/token/PrintTokenHandler.java | 2 +-
.../web/ozShell/volume/AddAclVolumeHandler.java | 5 +-
.../web/ozShell/volume/GetAclVolumeHandler.java | 4 +-
.../web/ozShell/volume/RemoveAclVolumeHandler.java | 7 +-
.../web/ozShell/volume/SetAclVolumeHandler.java | 5 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 3 +
hadoop-ozone/ozonefs-lib-current/pom.xml | 3 +
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 96 ++++
.../recon/recovery/ReconOmMetadataManagerImpl.java | 21 +-
.../spi/impl/ContainerDBServiceProviderImpl.java | 28 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 33 +-
.../recon/spi/impl/ReconContainerDBProvider.java | 32 +-
.../ozone/recon/AbstractOMMetadataManagerTest.java | 2 +-
.../apache/hadoop/ozone/recon/TestReconUtils.java | 75 ++-
.../recovery/TestReconOmMetadataManagerImpl.java | 133 +++--
.../impl/TestOzoneManagerServiceProviderImpl.java | 35 +-
.../spi/impl/TestReconContainerDBProvider.java | 13 -
91 files changed, 2772 insertions(+), 470 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 12/18: Merge remote-tracking branch 'upstream/trunk'
into HDDS-2181
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit a1adf87f4bf73b3421b23d97cdf14a80f58da58a
Merge: 251a4a7 70cf448
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Tue Oct 8 17:55:17 2019 -0700
Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 ++-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 67 ++++-----
.../hdds/scm/client/ContainerOperationClient.java | 10 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 40 +++---
.../hadoop/hdds/scm/storage/CommitWatcher.java | 8 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 6 +-
.../apache/hadoop/hdds/conf/HddsConfServlet.java | 4 +-
.../hdds/ratis/ContainerCommandRequestMessage.java | 107 +++++++++++++++
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 17 ++-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 21 ++-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 12 +-
.../hdds/security/token/BlockTokenVerifier.java | 4 +-
.../security/token/OzoneBlockTokenSelector.java | 8 +-
.../authority/PKIProfiles/DefaultProfile.java | 4 +-
.../hdds/security/x509/keys/HDDSKeyGenerator.java | 6 +-
.../apache/hadoop/hdds/tracing/StringCodec.java | 4 +-
.../hadoop/hdds/utils/BackgroundService.java | 9 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 6 +-
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 20 ++-
.../hadoop/ozone/common/ChecksumByteBuffer.java | 24 ++--
.../helpers/ContainerCommandRequestPBHelper.java | 16 ++-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +-
.../apache/hadoop/ozone/lease/LeaseManager.java | 6 +-
.../ratis/TestContainerCommandRequestMessage.java | 152 +++++++++++++++++++++
.../ozone/container/common/impl/ContainerSet.java | 64 ++++-----
.../container/common/impl/HddsDispatcher.java | 11 +-
.../RandomContainerDeletionChoosingPolicy.java | 11 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 13 +-
.../common/statemachine/EndpointStateMachine.java | 4 +-
.../CloseContainerCommandHandler.java | 6 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 28 ++--
.../server/ratis/ContainerStateMachine.java | 52 ++++---
.../transport/server/ratis/XceiverServerRatis.java | 5 +-
.../container/common/volume/HddsVolumeChecker.java | 14 +-
.../common/volume/ThrottledAsyncChecker.java | 8 +-
.../container/keyvalue/KeyValueBlockIterator.java | 6 +-
.../container/keyvalue/KeyValueContainerCheck.java | 34 ++---
.../ozone/container/keyvalue/KeyValueHandler.java | 84 ++++++++----
.../container/keyvalue/helpers/ChunkUtils.java | 8 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 8 +-
.../container/keyvalue/impl/ChunkManagerImpl.java | 9 +-
.../background/BlockDeletingService.java | 6 +-
.../container/ozoneimpl/ContainerController.java | 6 +-
.../container/ozoneimpl/ContainerDataScanner.java | 50 ++++---
.../ozoneimpl/ContainerDataScrubberMetrics.java | 4 +-
.../ozoneimpl/ContainerMetadataScanner.java | 19 ++-
.../ContainerMetadataScrubberMetrics.java | 5 +-
.../ozoneimpl/ContainerScrubberConfiguration.java | 17 +++
.../ozone/container/ozoneimpl/OzoneContainer.java | 8 +-
.../container/common/impl/TestContainerSet.java | 18 ++-
.../keyvalue/TestKeyValueContainerCheck.java | 69 ++++------
.../ozoneimpl/TestContainerScrubberMetrics.java | 25 ++--
.../container/ozoneimpl/TestOzoneContainer.java | 23 ++--
.../hadoop/hdds/server/events/EventQueue.java | 2 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 10 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 12 +-
.../scm/command/CommandStatusReportHandler.java | 12 +-
.../container/AbstractContainerReportHandler.java | 6 +-
.../scm/container/ContainerActionsHandler.java | 6 +-
.../hdds/scm/container/ContainerStateManager.java | 4 +-
.../IncrementalContainerReportHandler.java | 6 +-
.../algorithms/SCMContainerPlacementRackAware.java | 6 +-
.../scm/container/states/ContainerAttribute.java | 22 ++-
.../scm/container/states/ContainerStateMap.java | 6 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 10 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 4 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +-
.../hdds/scm/server/StorageContainerManager.java | 18 +--
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 118 +++++++++++++---
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
hadoop-ozone/dist/src/main/dockerbin/entrypoint.sh | 2 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 13 ++
.../container/common/TestBlockDeletingService.java | 24 ++--
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 7 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 13 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 38 +++---
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 12 +-
.../om/request/bucket/OMBucketCreateRequest.java | 12 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 16 ++-
.../request/bucket/OMBucketSetPropertyRequest.java | 13 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 6 +-
.../om/request/file/OMDirectoryCreateRequest.java | 4 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 6 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 4 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 4 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 4 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 6 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 4 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 22 +--
.../request/s3/bucket/S3BucketDeleteRequest.java | 13 +-
.../S3InitiateMultipartUploadRequest.java | 4 +-
.../multipart/S3MultipartUploadAbortRequest.java | 4 +-
.../S3MultipartUploadCommitPartRequest.java | 4 +-
.../S3MultipartUploadCompleteRequest.java | 4 +-
.../om/request/s3/security/S3GetSecretRequest.java | 6 +-
.../om/request/volume/OMVolumeCreateRequest.java | 10 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 10 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 59 ++++----
106 files changed, 1186 insertions(+), 605 deletions(-)
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index a754f56,20b5174..79500cc
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@@ -178,11 -177,10 +178,11 @@@ public class OMFileCreateRequest extend
OMClientResponse omClientResponse = null;
try {
// check Acl
- checkBucketAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.CREATE);
// acquire lock
- acquiredLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
+ acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
volumeName, bucketName);
OmBucketInfo bucketInfo =
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index c9fdb2d,196d61c..3fe5206
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@@ -140,8 -127,8 +140,8 @@@ public class OMKeyCommitRequest extend
String dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName,
keyName, commitKeyRequest.getClientID());
- bucketLockAcquired = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
- volumeName, bucketName);
- omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, volumeName,
- bucketName);
++ bucketLockAcquired = omMetadataManager.getLock()
++ .acquireWriteLock(BUCKET_LOCK, volumeName, bucketName);
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
omKeyInfo = omMetadataManager.getOpenKeyTable().get(dbOpenKey);
@@@ -179,11 -166,8 +179,11 @@@
ozoneManagerDoubleBufferHelper.add(omClientResponse,
transactionLogIndex));
}
- omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName,
- bucketName);
+
+ if(bucketLockAcquired) {
- omMetadataManager.getLock().releaseLock(BUCKET_LOCK, volumeName,
++ omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName,
+ bucketName);
+ }
}
// Performing audit logging outside of the lock.
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 05e7396,baa13ad..5229e81
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@@ -163,10 -162,9 +163,10 @@@ public class OMKeyCreateRequest extend
OMClientResponse omClientResponse = null;
try {
// check Acl
- checkBucketAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.CREATE);
- acquireLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
+ acquireLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
volumeName, bucketName);
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
//TODO: We can optimize this get here, if getKmsProvider is null, then
diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
index 7df1df8,526473c..c594120
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
@@@ -118,14 -117,10 +118,14 @@@ public class OMKeyRenameRequest extend
throw new OMException("Key name is empty",
OMException.ResultCodes.INVALID_KEY_NAME);
}
- // check Acl
- checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName);
+ // check Acls to see if user has access to perform delete operation on
+ // old key and create operation on new key
+ checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName,
+ IAccessAuthorizer.ACLType.DELETE);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, toKeyName,
+ IAccessAuthorizer.ACLType.CREATE);
- acquiredLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
+ acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
volumeName, bucketName);
// Not doing bucket/volume checks here. In this way we can avoid db
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 14/18: Fix unit test failures
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 076d05ca473100a3cb8247adcce42dd930231bcb
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Wed Oct 9 16:08:52 2019 -0700
Fix unit test failures
---
.../main/java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../hadoop/ozone/security/acl/IAccessAuthorizer.java | 2 +-
.../org/apache/hadoop/ozone/security/acl/OzoneObj.java | 1 +
.../ozone/security/acl/TestOzoneNativeAuthorizer.java | 5 ++++-
.../java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 10 ++++++++--
.../ozone/om/request/file/OMDirectoryCreateRequest.java | 3 ++-
.../hadoop/ozone/om/request/file/OMFileCreateRequest.java | 3 ++-
.../ozone/om/request/key/OMAllocateBlockRequest.java | 15 +++++++++++----
.../hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 15 +++++++++++----
.../hadoop/ozone/om/request/key/OMKeyCreateRequest.java | 3 ++-
.../hadoop/ozone/om/request/key/OMKeyDeleteRequest.java | 3 ++-
.../hadoop/ozone/om/request/key/OMKeyRenameRequest.java | 5 +++--
.../apache/hadoop/ozone/om/request/key/OMKeyRequest.java | 6 +++---
13 files changed, 51 insertions(+), 21 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 9817d87..7c8eb69 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -237,6 +237,7 @@ public final class OzoneConsts {
public static final String VOLUME = "volume";
public static final String BUCKET = "bucket";
public static final String KEY = "key";
+ public static final String OPEN_KEY = "openKey";
public static final String QUOTA = "quota";
public static final String QUOTA_IN_BYTES = "quotaInBytes";
public static final String OBJECT_ID = "objectID";
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/IAccessAuthorizer.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/IAccessAuthorizer.java
index d8a2660..939f2c1 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/IAccessAuthorizer.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/IAccessAuthorizer.java
@@ -64,7 +64,7 @@ public interface IAccessAuthorizer {
public static ACLType getAclTypeFromOrdinal(int ordinal) {
if (ordinal > length - 1 && ordinal > -1) {
- throw new IllegalArgumentException("Ordinal greater than array lentgh" +
+ throw new IllegalArgumentException("Ordinal greater than array length" +
". ordinal:" + ordinal);
}
return vals[ordinal];
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java
index 4a95e55..1d05ede 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java
@@ -95,6 +95,7 @@ public abstract class OzoneObj implements IOzoneObj {
VOLUME(OzoneConsts.VOLUME),
BUCKET(OzoneConsts.BUCKET),
KEY(OzoneConsts.KEY),
+ OPEN_KEY(OzoneConsts.OPEN_KEY),
PREFIX(OzoneConsts.PREFIX);
/**
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
index 43ce679..bedd959 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
@@ -69,6 +69,7 @@ import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentity
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.WORLD;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL;
+import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.CREATE;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.NONE;
import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.BUCKET;
import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.KEY;
@@ -362,6 +363,7 @@ public class TestOzoneNativeAuthorizer {
aclsToBeAdded.remove(NONE);
aclsToBeAdded.remove(ALL);
+ aclsToBeAdded.remove(CREATE);
// Fetch acls again.
for (ACLType a2 : aclsToBeAdded) {
@@ -410,7 +412,7 @@ public class TestOzoneNativeAuthorizer {
builder.setAclRights(a2).build()));
aclsToBeValidated.remove(a2);
for (ACLType a3 : aclsToBeValidated) {
- if (!a3.equals(a1) && !a3.equals(a2)) {
+ if (!a3.equals(a1) && !a3.equals(a2) && !a3.equals(CREATE)) {
assertFalse("User shouldn't have right " + a3 + ". " +
"Current acl rights for user:" + a1 + "," + a2,
nativeAuthorizer.checkAccess(obj,
@@ -462,6 +464,7 @@ public class TestOzoneNativeAuthorizer {
builder) throws OMException {
List<ACLType> allAcls = new ArrayList<>(Arrays.asList(ACLType.values()));
allAcls.remove(NONE);
+ allAcls.remove(CREATE);
for (ACLType a : allAcls) {
assertFalse("User shouldn't have right " + a + ".",
nativeAuthorizer.checkAccess(obj, builder.setAclRights(a).build()));
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index f3ae9b1..faa65bb 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -123,6 +123,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.KEY;
+import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.OPEN_KEY;
import static org.apache.hadoop.util.Time.monotonicNow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1656,8 +1657,13 @@ public class KeyManagerImpl implements KeyManager {
validateBucket(volume, bucket);
OmKeyInfo keyInfo = null;
try {
- OzoneFileStatus fileStatus = getFileStatus(args);
- keyInfo = fileStatus.getKeyInfo();
+ if (ozObject.getResourceType() == OPEN_KEY) {
+ keyInfo = metadataManager.getOpenKeyTable().get(objectKey);
+ } else {
+ OzoneFileStatus fileStatus = getFileStatus(args);
+ keyInfo = fileStatus.getKeyInfo();
+ }
+
if (keyInfo == null) {
// the key does not exist, but it is a parent "dir" of some key
// let access be determined based on volume/bucket/prefix ACL
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
index 6e45171..aaac874 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -129,7 +130,7 @@ public class OMDirectoryCreateRequest extends OMKeyRequest {
try {
// check Acl
checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
- IAccessAuthorizer.ACLType.CREATE);
+ IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY);
// Check if this is the root of the filesystem.
if (keyName.length() == 0) {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index 79500cc..52af0a3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@ -32,6 +32,7 @@ import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -179,7 +180,7 @@ public class OMFileCreateRequest extends OMKeyRequest {
try {
// check Acl
checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
- IAccessAuthorizer.ACLType.CREATE);
+ IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY);
// acquire lock
acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index a6702b3..ef2af6d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -176,12 +177,18 @@ public class OMAllocateBlockRequest extends OMKeyRequest {
// write ACL on key. Add client id to key name if ozone native
// authorizer is configured.
Configuration config = ozoneManager.getConfiguration();
- String keyNameForAclCheck = keyName;
if (OmUtils.isNativeAuthorizerEnabled(config)) {
- keyNameForAclCheck = keyName + "/" + allocateBlockRequest.getClientID();
+ String keyNameForAclCheck =
+ keyName + "/" + allocateBlockRequest.getClientID();
+ // During allocate block request, it is possible that key is
+ // not present in the key table and hence setting the resource type
+ // to OPEN_KEY to check the openKeyTable.
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyNameForAclCheck,
+ IAccessAuthorizer.ACLType.WRITE, OzoneObj.ResourceType.OPEN_KEY);
+ } else {
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.WRITE, OzoneObj.ResourceType.KEY);
}
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyNameForAclCheck,
- IAccessAuthorizer.ACLType.WRITE);
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
validateBucketAndVolume(omMetadataManager, volumeName,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 3fe5206..63ea5a0 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -123,12 +124,18 @@ public class OMKeyCommitRequest extends OMKeyRequest {
// write ACL on key. Add client id to key name if ozone native
// authorizer is configured.
Configuration config = ozoneManager.getConfiguration();
- String keyNameForAclCheck = keyName;
if (OmUtils.isNativeAuthorizerEnabled(config)) {
- keyNameForAclCheck = keyName + "/" + commitKeyRequest.getClientID();
+ String keyNameForAclCheck =
+ keyName + "/" + commitKeyRequest.getClientID();
+ // During key commit request, it is possible that key is
+ // not present in the key table and hence setting the resource type
+ // to OPEN_KEY to check the openKeyTable.
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyNameForAclCheck,
+ IAccessAuthorizer.ACLType.WRITE, OzoneObj.ResourceType.OPEN_KEY);
+ } else {
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.WRITE, OzoneObj.ResourceType.KEY);
}
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyNameForAclCheck,
- IAccessAuthorizer.ACLType.WRITE);
List<OmKeyLocationInfo> locationInfoList = commitKeyArgs
.getKeyLocationsList().stream()
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 5229e81..9681b20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -27,6 +27,7 @@ import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -164,7 +165,7 @@ public class OMKeyCreateRequest extends OMKeyRequest {
try {
// check Acl
checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
- IAccessAuthorizer.ACLType.CREATE);
+ IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY);
acquireLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
volumeName, bucketName);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
index 97c2554..28dfaa5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
@@ -24,6 +24,7 @@ import java.util.Map;
import com.google.common.base.Optional;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,7 +112,7 @@ public class OMKeyDeleteRequest extends OMKeyRequest {
try {
// check Acl
checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
- IAccessAuthorizer.ACLType.DELETE);
+ IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY);
String objectKey = omMetadataManager.getOzoneKey(
volumeName, bucketName, keyName);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
index c594120..6f7ff60 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
@@ -25,6 +25,7 @@ import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,9 +122,9 @@ public class OMKeyRenameRequest extends OMKeyRequest {
// check Acls to see if user has access to perform delete operation on
// old key and create operation on new key
checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName,
- IAccessAuthorizer.ACLType.DELETE);
+ IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY);
checkKeyAcls(ozoneManager, volumeName, bucketName, toKeyName,
- IAccessAuthorizer.ACLType.CREATE);
+ IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY);
acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK,
volumeName, bucketName);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 9520863..16e97e8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -526,11 +526,11 @@ public abstract class OMKeyRequest extends OMClientRequest {
* @throws IOException
*/
protected void checkKeyAcls(OzoneManager ozoneManager, String volume,
- String bucket, String key, IAccessAuthorizer.ACLType aclType)
+ String bucket, String key, IAccessAuthorizer.ACLType aclType,
+ OzoneObj.ResourceType resourceType)
throws IOException {
if (ozoneManager.getAclsEnabled()) {
- checkAcls(ozoneManager, OzoneObj.ResourceType.KEY,
- OzoneObj.StoreType.OZONE, aclType,
+ checkAcls(ozoneManager, resourceType, OzoneObj.StoreType.OZONE, aclType,
volume, bucket, key);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 03/18: Add delete acl to key rename request
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit a837ada0639a4e8eb33235eda19515a4a21c3a57
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Sep 26 14:22:20 2019 -0700
Add delete acl to key rename request
---
.../org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
index c763d00..7df1df8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
@@ -118,7 +118,10 @@ public class OMKeyRenameRequest extends OMKeyRequest {
throw new OMException("Key name is empty",
OMException.ResultCodes.INVALID_KEY_NAME);
}
- // check Acl
+ // check Acls to see if user has access to perform delete operation on
+ // old key and create operation on new key
+ checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName,
+ IAccessAuthorizer.ACLType.DELETE);
checkKeyAcls(ozoneManager, volumeName, bucketName, toKeyName,
IAccessAuthorizer.ACLType.CREATE);
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 01/18: HDDS-2181. Ozone Manager should send correct
ACL type in ACL requests to Authorizer
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit fe572e67669b632830798b77964b74407e7a8616
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Wed Sep 25 23:24:01 2019 -0700
HDDS-2181. Ozone Manager should send correct ACL type in ACL requests to Authorizer
---
.../hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java | 2 +-
.../hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java | 4 +++-
.../hadoop/ozone/om/request/file/OMFileCreateRequest.java | 4 +++-
.../hadoop/ozone/om/request/key/OMAllocateBlockRequest.java | 4 +++-
.../apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java | 4 +++-
.../apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java | 4 +++-
.../apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java | 4 +++-
.../apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java | 4 +++-
.../org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java | 10 ++++++----
9 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
index 06ebcc5..2dc0831 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java
@@ -143,7 +143,7 @@ public class OMBucketCreateRequest extends OMClientRequest {
try {
// check Acl
if (ozoneManager.getAclsEnabled()) {
- checkAcls(ozoneManager, OzoneObj.ResourceType.VOLUME,
+ checkAcls(ozoneManager, OzoneObj.ResourceType.BUCKET,
OzoneObj.StoreType.OZONE, IAccessAuthorizer.ACLType.CREATE,
volumeName, bucketName, null);
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
index 1c39433..3f53e54 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -127,7 +128,8 @@ public class OMDirectoryCreateRequest extends OMKeyRequest {
OMClientResponse omClientResponse = null;
try {
// check Acl
- checkBucketAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.CREATE);
// Check if this is the root of the filesystem.
if (keyName.length() == 0) {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
index b51a4d6..a754f56 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java
@@ -31,6 +31,7 @@ import javax.annotation.Nonnull;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -177,7 +178,8 @@ public class OMFileCreateRequest extends OMKeyRequest {
OMClientResponse omClientResponse = null;
try {
// check Acl
- checkBucketAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.CREATE);
// acquire lock
acquiredLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index e800927..df565de 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -26,6 +26,7 @@ import java.util.Map;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -169,7 +170,8 @@ public class OMAllocateBlockRequest extends OMKeyRequest {
OmKeyInfo omKeyInfo = null;
try {
// check Acl
- checkBucketAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.WRITE);
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
validateBucketAndVolume(omMetadataManager, volumeName,
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 69e5405..622deb8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -26,6 +26,7 @@ import java.util.stream.Collectors;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -115,7 +116,8 @@ public class OMKeyCommitRequest extends OMKeyRequest {
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
try {
// check Acl
- checkBucketAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.CREATE);
List<OmKeyLocationInfo> locationInfoList = commitKeyArgs
.getKeyLocationsList().stream()
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 2596646..05e7396 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -26,6 +26,7 @@ import java.util.stream.Collectors;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -162,7 +163,8 @@ public class OMKeyCreateRequest extends OMKeyRequest {
OMClientResponse omClientResponse = null;
try {
// check Acl
- checkBucketAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.CREATE);
acquireLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
volumeName, bucketName);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
index eb366ad..0b9b1cb 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java
@@ -23,6 +23,7 @@ import java.util.Map;
import com.google.common.base.Optional;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -109,7 +110,8 @@ public class OMKeyDeleteRequest extends OMKeyRequest {
OMClientResponse omClientResponse = null;
try {
// check Acl
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ IAccessAuthorizer.ACLType.DELETE);
String objectKey = omMetadataManager.getOzoneKey(
volumeName, bucketName, keyName);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
index eb8a59e..c763d00 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java
@@ -24,6 +24,7 @@ import java.util.Map;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,7 +119,8 @@ public class OMKeyRenameRequest extends OMKeyRequest {
OMException.ResultCodes.INVALID_KEY_NAME);
}
// check Acl
- checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName);
+ checkKeyAcls(ozoneManager, volumeName, bucketName, toKeyName,
+ IAccessAuthorizer.ACLType.CREATE);
acquiredLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK,
volumeName, bucketName);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 8e1e760..9520863 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -507,10 +507,11 @@ public abstract class OMKeyRequest extends OMClientRequest {
* @throws IOException
*/
protected void checkBucketAcls(OzoneManager ozoneManager, String volume,
- String bucket, String key) throws IOException {
+ String bucket, String key, IAccessAuthorizer.ACLType aclType)
+ throws IOException {
if (ozoneManager.getAclsEnabled()) {
checkAcls(ozoneManager, OzoneObj.ResourceType.BUCKET,
- OzoneObj.StoreType.OZONE, IAccessAuthorizer.ACLType.WRITE,
+ OzoneObj.StoreType.OZONE, aclType,
volume, bucket, key);
}
}
@@ -525,10 +526,11 @@ public abstract class OMKeyRequest extends OMClientRequest {
* @throws IOException
*/
protected void checkKeyAcls(OzoneManager ozoneManager, String volume,
- String bucket, String key) throws IOException {
+ String bucket, String key, IAccessAuthorizer.ACLType aclType)
+ throws IOException {
if (ozoneManager.getAclsEnabled()) {
checkAcls(ozoneManager, OzoneObj.ResourceType.KEY,
- OzoneObj.StoreType.OZONE, IAccessAuthorizer.ACLType.WRITE,
+ OzoneObj.StoreType.OZONE, aclType,
volume, bucket, key);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 04/18: Merge remote-tracking branch 'upstream/trunk'
into HDDS-2181
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit cfabf1ab287beb1cde4a604b112a0fffcc5c8135
Merge: a837ada 1a93d9d
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Fri Sep 27 13:30:44 2019 -0700
Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
hadoop-hdds/common/pom.xml | 4 +-
.../hadoop/hdds/conf/ConfigFileGenerator.java | 5 +-
hadoop-hdds/container-service/pom.xml | 9 +-
hadoop-hdds/pom.xml | 15 ---
hadoop-hdds/server-scm/pom.xml | 4 +-
.../hdds/scm/container/SCMContainerManager.java | 31 +++++-
.../metrics/SCMContainerManagerMetrics.java | 90 +++++++++++++++++
.../hdds/scm/container/metrics/package-info.java | 22 ++++
hadoop-ozone/common/pom.xml | 4 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 40 ++++++--
.../src/main/proto/OzoneManagerProtocol.proto | 2 +
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 12 ++-
hadoop-ozone/dist/src/main/compose/testlib.sh | 15 ++-
hadoop-ozone/insight/pom.xml | 9 +-
.../metrics/TestSCMContainerManagerMetrics.java | 112 +++++++++++++++++++++
.../client/rpc/TestOzoneRpcClientAbstract.java | 76 +++++++++++++-
hadoop-ozone/ozone-manager/pom.xml | 5 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 49 ++++-----
.../request/s3/bucket/S3BucketCreateRequest.java | 3 +-
.../om/request/volume/OMVolumeCreateRequest.java | 2 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 3 +-
.../ozone/om/request/volume/OMVolumeRequest.java | 18 +++-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 1 -
.../ozone/om/response/key/OMKeyCommitResponse.java | 3 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 16 +--
.../ozone/om/response/key/OMKeyPurgeResponse.java | 3 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 4 +-
.../multipart/S3MultipartUploadAbortResponse.java | 12 +--
.../S3MultipartUploadCommitPartResponse.java | 34 +++----
.../ozone/om/request/TestOMRequestUtils.java | 19 ++--
.../ozone/om/response/TestOMResponseUtils.java | 2 +
.../volume/TestOMVolumeCreateResponse.java | 1 +
.../volume/TestOMVolumeDeleteResponse.java | 5 +-
.../volume/TestOMVolumeSetOwnerResponse.java | 9 +-
hadoop-ozone/ozonefs-lib-current/pom.xml | 4 +-
hadoop-ozone/ozonefs-lib-legacy/pom.xml | 4 +-
hadoop-ozone/ozonefs/pom.xml | 5 +-
hadoop-ozone/pom.xml | 8 --
hadoop-ozone/recon/pom.xml | 4 +-
hadoop-ozone/s3gateway/pom.xml | 5 +-
hadoop-ozone/tools/pom.xml | 9 +-
hadoop-ozone/upgrade/pom.xml | 5 +-
pom.ozone.xml | 29 +++---
45 files changed, 540 insertions(+), 182 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 10/18: Fix unit test failures
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 00a116033c426fa950facffe07044bda37664861
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Fri Oct 4 15:04:37 2019 -0700
Fix unit test failures
---
.../java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index 92d6cdb..f634ff3 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@ -100,6 +100,7 @@ public class TestOMKeyRequest {
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
when(ozoneManager.getMetrics()).thenReturn(omMetrics);
when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
+ when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
auditLogger = Mockito.mock(AuditLogger.class);
when(ozoneManager.getAuditLogger()).thenReturn(auditLogger);
Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 16/18: Fix acceptance test failures
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit ac4990fc02da8e6359a0099f38699940805afc5a
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Oct 10 19:07:47 2019 -0700
Fix acceptance test failures
---
.../java/org/apache/hadoop/ozone/security/acl/OzoneObjInfo.java | 1 +
.../src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 9 +++++++--
.../apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java | 1 +
3 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObjInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObjInfo.java
index cbae18c..f29f372 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObjInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObjInfo.java
@@ -62,6 +62,7 @@ public final class OzoneObjInfo extends OzoneObj {
return OZONE_URI_DELIMITER + getVolumeName()
+ OZONE_URI_DELIMITER + getBucketName();
case KEY:
+ case OPEN_KEY:
return OZONE_URI_DELIMITER + getVolumeName()
+ OZONE_URI_DELIMITER + getBucketName()
+ OZONE_URI_DELIMITER + getKeyName();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index d0be40b..19976e5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -1660,8 +1660,13 @@ public class KeyManagerImpl implements KeyManager {
if (ozObject.getResourceType() == OPEN_KEY) {
keyInfo = metadataManager.getOpenKeyTable().get(objectKey);
} else {
- OzoneFileStatus fileStatus = getFileStatus(args);
- keyInfo = fileStatus.getKeyInfo();
+ try {
+ OzoneFileStatus fileStatus = getFileStatus(args);
+ keyInfo = fileStatus.getKeyInfo();
+ } catch (Exception e) {
+ throw new OMException("Key not found, checkAccess failed. Key:" +
+ objectKey, KEY_NOT_FOUND);
+ }
}
if (keyInfo == null) {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
index 1731421..d974537 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java
@@ -108,6 +108,7 @@ public class OzoneNativeAuthorizer implements IAccessAuthorizer {
return (bucketAccess
&& volumeManager.checkAccess(objInfo, parentContext));
case KEY:
+ case OPEN_KEY:
LOG.trace("Checking access for Key:" + objInfo);
// Skip key access check for CREATE acl since
// key will not exist at the time of creation
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 05/18: Merge remote-tracking branch 'upstream/trunk'
into HDDS-2181
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 4fad1bd6d73b7ca18b28bf588aa9d278e6ef3021
Merge: cfabf1a 5249f99
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Thu Oct 3 14:56:32 2019 -0700
Merge remote-tracking branch 'upstream/trunk' into HDDS-2181
.../SCMSecurityProtocolClientSideTranslatorPB.java | 104 +++--
.../SCMSecurityProtocolServerSideTranslatorPB.java | 132 ------
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 +
...inerLocationProtocolClientSideTranslatorPB.java | 411 +++++++++---------
.../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +-
.../hadoop/hdds/utils/db/cache/TableCache.java | 9 +
.../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 7 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +
.../src/main/proto/SCMSecurityProtocol.proto | 96 +++--
.../src/main/proto/ScmBlockLocationProtocol.proto | 2 +-
.../proto/StorageContainerLocationProtocol.proto | 185 ++++----
.../common/src/main/resources/ozone-default.xml | 8 +
.../server/ratis/ContainerStateMachine.java | 20 +-
.../transport/server/ratis/XceiverServerRatis.java | 8 +-
.../replication/GrpcReplicationClient.java | 6 +
...inerDatanodeProtocolClientSideTranslatorPB.java | 60 +--
...inerDatanodeProtocolServerSideTranslatorPB.java | 115 ++---
.../proto/StorageContainerDatanodeProtocol.proto | 58 ++-
.../ozone/container/common/SCMTestUtils.java | 4 +-
.../hadoop/hdds/server/PrometheusMetricsSink.java | 16 +-
.../hdds/server/TestPrometheusMetricsSink.java | 77 +++-
hadoop-hdds/pom.xml | 2 +-
.../container/AbstractContainerReportHandler.java | 8 +
.../hdds/scm/container/ContainerManager.java | 8 +
.../hdds/scm/container/ContainerReportHandler.java | 2 +
.../IncrementalContainerReportHandler.java | 10 +
.../hdds/scm/container/SCMContainerManager.java | 17 +
.../metrics/SCMContainerManagerMetrics.java | 54 +++
.../SCMSecurityProtocolServerSideTranslatorPB.java | 186 ++++++++
...inerLocationProtocolServerSideTranslatorPB.java | 476 +++++++++++----------
.../hdds/scm/server/SCMClientProtocolServer.java | 12 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 102 ++---
.../hdds/scm/server/SCMSecurityProtocolServer.java | 27 +-
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 22 +-
.../hdds/scm/cli/container/CloseSubcommand.java | 7 +-
...CloseSubcommand.java => ContainerCommands.java} | 37 +-
.../hdds/scm/cli/container/CreateSubcommand.java | 5 +-
.../hdds/scm/cli/container/DeleteSubcommand.java | 7 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 5 +-
.../hdds/scm/cli/container/ListSubcommand.java | 5 +-
.../cli/pipeline/ActivatePipelineSubcommand.java | 11 +-
.../scm/cli/pipeline/ClosePipelineSubcommand.java | 11 +-
.../cli/pipeline/DeactivatePipelineSubcommand.java | 11 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 11 +-
.../PipelineCommands.java} | 38 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 3 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 19 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 4 +-
...lumeListCodec.java => UserVolumeInfoCodec.java} | 13 +-
.../hadoop/ozone/security/GDPRSymmetricKey.java | 8 +-
.../OzoneDelegationTokenSecretManager.java | 5 +-
.../hadoop/ozone/security/OzoneSecretManager.java | 7 +-
.../src/main/proto/OzoneManagerProtocol.proto | 2 +-
.../ozone/security/TestGDPRSymmetricKey.java | 4 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +-
.../src/main/compose/ozonesecure-mr/docker-config | 3 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 4 +-
hadoop-ozone/dist/src/main/compose/test-single.sh | 2 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 12 +-
.../ozone/insight/BaseInsightSubCommand.java | 7 +-
.../scm/ScmProtocolBlockLocationInsight.java | 4 +-
...va => ScmProtocolContainerLocationInsight.java} | 18 +-
...nsight.java => ScmProtocolDatanodeInsight.java} | 27 +-
...nsight.java => ScmProtocolSecurityInsight.java} | 18 +-
.../metrics/TestSCMContainerManagerMetrics.java | 63 ++-
.../rpc/TestContainerStateMachineFailures.java | 65 +++
.../ozone/om/TestOzoneManagerConfiguration.java | 9 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 25 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 232 ++--------
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 35 +-
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 306 +++++++++++++
.../hadoop/ozone/om/{ => ha}/OMNodeDetails.java | 2 +-
.../apache/hadoop/ozone/om/ha/package-info.java} | 23 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 5 +-
.../om/request/volume/OMVolumeCreateRequest.java | 5 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 2 +-
.../ozone/om/request/volume/OMVolumeRequest.java | 14 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 4 +-
.../om/response/volume/OMVolumeCreateResponse.java | 11 +-
.../om/response/volume/OMVolumeDeleteResponse.java | 8 +-
.../response/volume/OMVolumeSetOwnerResponse.java | 8 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../ozone/om/request/TestOMRequestUtils.java | 6 +-
.../hadoop/ozone/om/request/package-info.java} | 18 +-
.../request/volume/TestOMVolumeCreateRequest.java | 6 +-
.../volume/TestOMVolumeSetOwnerRequest.java | 4 +-
.../ozone/om/response/TestOMResponseUtils.java | 6 +-
.../hadoop/ozone/om/response/package-info.java} | 18 +-
.../volume/TestOMVolumeCreateResponse.java | 6 +-
.../volume/TestOMVolumeDeleteResponse.java | 8 +-
.../volume/TestOMVolumeSetOwnerResponse.java | 10 +-
.../ozone/om/response/volume/package-info.java} | 29 +-
.../TestOzoneDelegationTokenSecretManager.java | 29 +-
hadoop-ozone/pom.xml | 6 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 11 +-
.../hadoop/ozone/s3/OzoneServiceProvider.java | 50 ++-
.../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 44 ++
.../hadoop/ozone/s3/util/TestOzoneS3Util.java | 130 ++++++
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 4 +-
.../services/org.apache.hadoop.fs.FileSystem | 16 +
102 files changed, 2326 insertions(+), 1464 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org
[hadoop-ozone] 13/18: Handle acl checks correctly in allocate block
request
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2181
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit f926998cc29459b8370cc1f094681e6a9dd30df1
Author: Vivek Ratnavel Subramanian <vi...@gmail.com>
AuthorDate: Tue Oct 8 17:56:48 2019 -0700
Handle acl checks correctly in allocate block request
---
.../OzoneManagerProtocolClientSideTranslatorPB.java | 2 +-
.../hadoop/ozone/om/request/key/OMAllocateBlockRequest.java | 12 +++++++++++-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index c9dc8ec..ee9e19a 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -340,7 +340,7 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
if (omResponse.hasLeaderOMNodeId() && omFailoverProxyProvider != null) {
String leaderOmId = omResponse.getLeaderOMNodeId();
- // Failover to the OM node returned by OMReponse leaderOMNodeId if
+ // Failover to the OM node returned by OMResponse leaderOMNodeId if
// current proxy is not pointing to that node.
omFailoverProxyProvider.performFailoverIfRequired(leaderOmId);
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
index df565de..a6702b3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java
@@ -25,6 +25,8 @@ import java.util.Map;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.util.Time;
@@ -170,7 +172,15 @@ public class OMAllocateBlockRequest extends OMKeyRequest {
OmKeyInfo omKeyInfo = null;
try {
// check Acl
- checkKeyAcls(ozoneManager, volumeName, bucketName, keyName,
+ // Native authorizer requires client id as part of keyname to check
+ // write ACL on key. Add client id to key name if ozone native
+ // authorizer is configured.
+ Configuration config = ozoneManager.getConfiguration();
+ String keyNameForAclCheck = keyName;
+ if (OmUtils.isNativeAuthorizerEnabled(config)) {
+ keyNameForAclCheck = keyName + "/" + allocateBlockRequest.getClientID();
+ }
+ checkKeyAcls(ozoneManager, volumeName, bucketName, keyNameForAclCheck,
IAccessAuthorizer.ACLType.WRITE);
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org