You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/08/29 18:02:56 UTC
hadoop git commit: HADOOP-14802. Add support for using container
saskeys for all accesses. Contributed by Sivaguru Sankaridurg
Repository: hadoop
Updated Branches:
refs/heads/trunk 1f6dc4ee9 -> 021974f4c
HADOOP-14802. Add support for using container saskeys for all accesses.
Contributed by Sivaguru Sankaridurg
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/021974f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/021974f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/021974f4
Branch: refs/heads/trunk
Commit: 021974f4cb1eb8cf530dabc95039fed46e66a43d
Parents: 1f6dc4e
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 29 14:58:43 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 29 19:02:43 2017 +0100
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 10 ++++-
.../conf/TestCommonConfigurationFields.java | 1 +
.../fs/azure/SecureStorageInterfaceImpl.java | 29 +++++++++----
.../hadoop-azure/src/site/markdown/index.md | 9 ++++
...stNativeAzureFSAuthWithBlobSpecificKeys.java | 44 ++++++++++++++++++++
5 files changed, 83 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/021974f4/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 7c4b0f1..cb061aa 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1380,7 +1380,15 @@
This flag is relevant only when fs.azure.authorization is enabled.
</description>
</property>
-
+<property>
+ <name>fs.azure.saskey.usecontainersaskeyforallaccess</name>
+ <value>true</value>
+ <description>
+ Use container saskey for access to all blobs within the container.
+ Blob-specific saskeys are not used when this setting is enabled.
+ This setting provides better performance compared to blob-specific saskeys.
+ </description>
+</property>
<property>
<name>io.seqfile.compress.blocksize</name>
<value>1000000</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/021974f4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index d0e0a35..3324886 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -123,6 +123,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPropsToSkipCompare.add("fs.azure.secure.mode");
xmlPropsToSkipCompare.add("fs.azure.authorization");
xmlPropsToSkipCompare.add("fs.azure.authorization.caching.enable");
+ xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess");
xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix");
// Deprecated properties. These should eventually be removed from the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/021974f4/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
index 3d33453..5dbb6bc 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
@@ -71,6 +71,13 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
private String storageAccount;
private RetryPolicyFactory retryPolicy;
private int timeoutIntervalInMs;
+ private boolean useContainerSasKeyForAllAccess;
+
+ /**
+ * Configuration key to specify if containerSasKey should be used for all accesses
+ */
+ public static final String KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS =
+ "fs.azure.saskey.usecontainersaskeyforallaccess";
public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode,
Configuration conf) throws SecureModeException {
@@ -88,6 +95,7 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
}
this.sasKeyGenerator = remoteSasKeyGenerator;
}
+ this.useContainerSasKeyForAllAccess = conf.getBoolean(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, true);
}
@Override
@@ -145,7 +153,9 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
if (timeoutIntervalInMs > 0) {
container.getServiceClient().getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs);
}
- return new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator);
+ return (useContainerSasKeyForAllAccess)
+ ? new SASCloudBlobContainerWrapperImpl(storageAccount, container, null)
+ : new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator);
} catch (SASKeyGenerationException sasEx) {
String errorMsg = "Encountered SASKeyGeneration exception while "
+ "generating SAS Key for container : " + name
@@ -226,12 +236,12 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
public CloudBlobWrapper getBlockBlobReference(String relativePath)
throws URISyntaxException, StorageException {
try {
- CloudBlockBlob blob = new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(
- storageAccount, getName(), relativePath));
+ CloudBlockBlob blob = (sasKeyGenerator!=null)
+ ? new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(), relativePath))
+ : container.getBlockBlobReference(relativePath);
blob.getServiceClient().setDefaultRequestOptions(
container.getServiceClient().getDefaultRequestOptions());
- return new SASCloudBlockBlobWrapperImpl(
- blob);
+ return new SASCloudBlockBlobWrapperImpl(blob);
} catch (SASKeyGenerationException sasEx) {
String errorMsg = "Encountered SASKeyGeneration exception while "
+ "generating SAS Key for relativePath : " + relativePath
@@ -245,12 +255,13 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
public CloudBlobWrapper getPageBlobReference(String relativePath)
throws URISyntaxException, StorageException {
try {
- CloudPageBlob blob = new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(
- storageAccount, getName(), relativePath));
+ CloudPageBlob blob = (sasKeyGenerator!=null)
+ ? new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(), relativePath))
+ : container.getPageBlobReference(relativePath);
+
blob.getServiceClient().setDefaultRequestOptions(
container.getServiceClient().getDefaultRequestOptions());
- return new SASCloudPageBlobWrapperImpl(
- blob);
+ return new SASCloudPageBlobWrapperImpl(blob);
} catch (SASKeyGenerationException sasEx) {
String errorMsg = "Encountered SASKeyGeneration exception while "
+ "generating SAS Key for relativePath : " + relativePath
http://git-wip-us.apache.org/repos/asf/hadoop/blob/021974f4/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 79cb0ea..758650d 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -476,6 +476,15 @@ The maximum number of entries that that cache can hold can be customized using t
</property>
```
+ Use container saskey for access to all blobs within the container.
+ Blob-specific saskeys are not used when this setting is enabled.
+ This setting provides better performance compared to blob-specific saskeys.
+ ```
+ <property>
+ <name>fs.azure.saskey.usecontainersaskeyforallaccess</name>
+ <value>true</value>
+ </property>
+```
## Testing the hadoop-azure Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests
http://git-wip-us.apache.org/repos/asf/hadoop/blob/021974f4/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
new file mode 100644
index 0000000..6149154
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import org.apache.hadoop.conf.Configuration;
+
+import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS;
+
+/**
+ * Test class to hold all WASB authorization tests that use blob-specific keys
+ * to access storage.
+ */
+public class TestNativeAzureFSAuthWithBlobSpecificKeys
+ extends TestNativeAzureFileSystemAuthorizationWithOwner {
+
+ @Override
+ public Configuration getConfiguration() {
+ Configuration conf = super.getConfiguration();
+ conf.set(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, "false");
+ return conf;
+ }
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ Configuration conf = getConfiguration();
+ return AzureBlobStorageTestAccount.create(conf);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org