You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2019/03/26 16:00:18 UTC
[hadoop] branch trunk updated: HDDS-939. Add S3 access check to
Ozone manager. Contributed by Ajay Kumar. (#634)
This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 82d4772 HDDS-939. Add S3 access check to Ozone manager. Contributed by Ajay Kumar. (#634)
82d4772 is described below
commit 82d477293c879ed4529efe2b5e8d138e09bbce3c
Author: Ajay Yadav <78...@users.noreply.github.com>
AuthorDate: Tue Mar 26 08:59:59 2019 -0700
HDDS-939. Add S3 access check to Ozone manager. Contributed by Ajay Kumar. (#634)
---
.../hadoop/ozone/om/S3SecretManagerImpl.java | 21 ++++++-------
.../hadoop/ozone/om/helpers/S3SecretValue.java | 11 ++-----
.../client/rpc/TestOzoneRpcClientAbstract.java | 20 ++++++++++++
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 7 +++--
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 7 +++--
.../apache/hadoop/ozone/s3/header/Credential.java | 16 ++++++++--
.../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 36 ++++++++++++++++++++++
.../hadoop/ozone/s3/endpoint/TestRootList.java | 4 ++-
8 files changed, 94 insertions(+), 28 deletions(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java
index 6febcaf..44712d5 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java
@@ -61,8 +61,7 @@ public class S3SecretManagerImpl implements S3SecretManager {
public S3SecretValue getS3Secret(String kerberosID) throws IOException {
Preconditions.checkArgument(Strings.isNotBlank(kerberosID),
"kerberosID cannot be null or empty.");
- String awsAccessKeyStr = DigestUtils.md5Hex(kerberosID);
- byte[] awsAccessKey = awsAccessKeyStr.getBytes(UTF_8);
+ byte[] awsAccessKey = kerberosID.getBytes(UTF_8);
S3SecretValue result = null;
omMetadataManager.getLock().acquireS3SecretLock(kerberosID);
try {
@@ -77,33 +76,31 @@ public class S3SecretManagerImpl implements S3SecretManager {
result = S3SecretValue.fromProtobuf(
OzoneManagerProtocolProtos.S3Secret.parseFrom(s3Secret));
}
- result.setAwsAccessKey(awsAccessKeyStr);
} finally {
omMetadataManager.getLock().releaseS3SecretLock(kerberosID);
}
- LOG.trace("Secret for kerberosID:{},accessKey:{}, proto:{}", kerberosID,
- awsAccessKeyStr, result);
+ LOG.trace("Secret for accessKey:{}, proto:{}", kerberosID, result);
return result;
}
@Override
- public String getS3UserSecretString(String awsAccessKeyId)
+ public String getS3UserSecretString(String kerberosID)
throws IOException {
- Preconditions.checkArgument(Strings.isNotBlank(awsAccessKeyId),
+ Preconditions.checkArgument(Strings.isNotBlank(kerberosID),
"awsAccessKeyId cannot be null or empty.");
- LOG.trace("Get secret for awsAccessKey:{}", awsAccessKeyId);
+ LOG.trace("Get secret for awsAccessKey:{}", kerberosID);
byte[] s3Secret;
- omMetadataManager.getLock().acquireS3SecretLock(awsAccessKeyId);
+ omMetadataManager.getLock().acquireS3SecretLock(kerberosID);
try {
s3Secret = omMetadataManager.getS3SecretTable()
- .get(awsAccessKeyId.getBytes(UTF_8));
+ .get(kerberosID.getBytes(UTF_8));
if (s3Secret == null) {
throw new OzoneSecurityException("S3 secret not found for " +
- "awsAccessKeyId " + awsAccessKeyId, S3_SECRET_NOT_FOUND);
+ "awsAccessKeyId " + kerberosID, S3_SECRET_NOT_FOUND);
}
} finally {
- omMetadataManager.getLock().releaseS3SecretLock(awsAccessKeyId);
+ omMetadataManager.getLock().releaseS3SecretLock(kerberosID);
}
return OzoneManagerProtocolProtos.S3Secret.parseFrom(s3Secret)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java
index 23f4c05..2608e77 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.ozone.om.helpers;
-import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
/**
@@ -26,12 +25,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
public class S3SecretValue {
private String kerberosID;
private String awsSecret;
- private String awsAccessKey;
public S3SecretValue(String kerberosID, String awsSecret) {
this.kerberosID = kerberosID;
this.awsSecret = awsSecret;
- this.awsAccessKey = DigestUtils.md5Hex(kerberosID);
}
public String getKerberosID() {
@@ -51,11 +48,7 @@ public class S3SecretValue {
}
public String getAwsAccessKey() {
- return awsAccessKey;
- }
-
- public void setAwsAccessKey(String awsAccessKey) {
- this.awsAccessKey = awsAccessKey;
+ return kerberosID;
}
public static S3SecretValue fromProtobuf(
@@ -72,6 +65,6 @@ public class S3SecretValue {
@Override
public String toString() {
- return "awsAccessKey=" + awsAccessKey + "\nawsSecret=" + awsSecret;
+ return "awsAccessKey=" + kerberosID + "\nawsSecret=" + awsSecret;
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 58a29ed..bd496d0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -81,6 +81,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
+import org.apache.hadoop.ozone.s3.util.OzoneS3Util;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.Time;
@@ -92,6 +93,8 @@ import org.apache.commons.lang3.RandomUtils;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.either;
import org.junit.Assert;
+
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
@@ -288,6 +291,23 @@ public abstract class TestOzoneRpcClientAbstract {
Assert.assertTrue(volume.getCreationTime() >= currentTime);
}
+ @Test
+ public void testCreateSecureS3Bucket() throws IOException {
+ long currentTime = Time.now();
+ String userName = "ozone/localhost@EXAMPLE.COM";
+ String bucketName = UUID.randomUUID().toString();
+ String s3VolumeName = OzoneS3Util.getVolumeName(userName);
+ store.createS3Bucket(s3VolumeName, bucketName);
+ String volumeName = store.getOzoneVolumeName(bucketName);
+ assertEquals(volumeName, "s3" + s3VolumeName);
+
+ OzoneVolume volume = store.getVolume(volumeName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ Assert.assertEquals(bucketName, bucket.getName());
+ Assert.assertTrue(bucket.getCreationTime() >= currentTime);
+ Assert.assertTrue(volume.getCreationTime() >= currentTime);
+ }
+
@Test
public void testListS3Buckets()
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index b9a29a6..b85b8d3 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -53,6 +53,8 @@ import org.apache.hadoop.ozone.s3.util.S3StorageType;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.lang3.StringUtils;
+
+import static org.apache.hadoop.ozone.s3.util.OzoneS3Util.getVolumeName;
import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
@@ -196,9 +198,10 @@ public class BucketEndpoint extends EndpointBase {
public Response put(@PathParam("bucket") String bucketName, @Context
HttpHeaders httpHeaders) throws IOException, OS3Exception {
- String userName = getAuthenticationHeaderParser().getAccessKeyID();
+ String volumeName = getVolumeName(getAuthenticationHeaderParser().
+ getAccessKeyID());
- String location = createS3Bucket(userName, bucketName);
+ String location = createS3Bucket(volumeName, bucketName);
LOG.info("Location is {}", location);
return Response.status(HttpStatus.SC_OK).header("Location", location)
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
index bb91840..23d02e9 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.ozone.s3.util.OzoneS3Util.getVolumeName;
+
/**
* Top level rest endpoint.
*/
@@ -63,8 +65,9 @@ public class RootEndpoint extends EndpointBase {
.header("Location", "/static/")
.build();
}
- String userName = authenticationHeaderParser.getAccessKeyID();
- Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(userName,
+ String volumeName = getVolumeName(authenticationHeaderParser.
+ getAccessKeyID());
+ Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(volumeName,
null);
while (bucketIterator.hasNext()) {
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java
index b53d41d..883980a 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java
@@ -59,13 +59,25 @@ public class Credential {
@SuppressWarnings("StringSplitter")
public void parseCredential() throws OS3Exception {
String[] split = credential.split("/");
- if (split.length == 5) {
+ switch (split.length) {
+ case 5:
+ // Ex: dkjad922329ddnks/20190321/us-west-1/s3/aws4_request
accessKeyID = split[0].trim();
date = split[1].trim();
awsRegion = split[2].trim();
awsService = split[3].trim();
awsRequest = split[4].trim();
- } else {
+ return;
+ case 6:
+ // Access id is kerberos principal.
+ // Ex: testuser/om@EXAMPLE.COM/20190321/us-west-1/s3/aws4_request
+ accessKeyID = split[0] + "/" +split[1];
+ date = split[2].trim();
+ awsRegion = split[3].trim();
+ awsService = split[4].trim();
+ awsRequest = split[5].trim();
+ return;
+ default:
LOG.error("Credentials not in expected format. credential:{}",
credential);
throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_HEADER, credential);
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java
new file mode 100644
index 0000000..129ea2d
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.ozone.s3.util;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import java.util.Objects;
+
+/**
+ * Ozone util for S3 related operations.
+ */
+public final class OzoneS3Util {
+
+ private OzoneS3Util() {
+ }
+
+ public static String getVolumeName(String userName) {
+ Objects.requireNonNull(userName);
+ return DigestUtils.md5Hex(userName);
+ }
+}
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
index 0636eaf..b7512cb 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser;
import static org.junit.Assert.assertEquals;
+import org.apache.hadoop.ozone.s3.util.OzoneS3Util;
import org.junit.Before;
import org.junit.Test;
@@ -61,10 +62,11 @@ public class TestRootList {
ListBucketResponse response =
(ListBucketResponse) rootEndpoint.get().getEntity();
assertEquals(0, response.getBucketsNum());
+ String volumeName = OzoneS3Util.getVolumeName(userName);
String bucketBaseName = "bucket-" + getClass().getName();
for(int i = 0; i < 10; i++) {
- objectStoreStub.createS3Bucket(userName, bucketBaseName + i);
+ objectStoreStub.createS3Bucket(volumeName, bucketBaseName + i);
}
response = (ListBucketResponse) rootEndpoint.get().getEntity();
assertEquals(10, response.getBucketsNum());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org