You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tm...@apache.org on 2020/06/18 02:09:16 UTC
[hadoop] branch trunk updated: HADOOP-17076: ABFS: Delegation SAS
Generator Updates Contributed by Thomas Marquardt.
This is an automated email from the ASF dual-hosted git repository.
tmarquardt pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new caf3995 HADOOP-17076: ABFS: Delegation SAS Generator Updates Contributed by Thomas Marquardt.
caf3995 is described below
commit caf3995ac2bbc3241896babb9a607272462f70ca
Author: Thomas Marquardt <tm...@microsoft.com>
AuthorDate: Wed Jun 17 23:12:22 2020 +0000
HADOOP-17076: ABFS: Delegation SAS Generator Updates
Contributed by Thomas Marquardt.
DETAILS:
1) The authentication version in the service has been updated from Dec19 to Feb20, so need to update the client.
2) Add support and test cases for getXattr and setXAttr.
3) Update DelegationSASGenerator and related to use Duration instead of int for time periods.
4) Cleanup DelegationSASGenerator switch/case statement that maps operations to permissions.
5) Cleanup SASGenerator classes to use String.equals instead of ==.
TESTS:
Added tests for getXAttr and setXAttr.
All tests are passing against my account in eastus2euap:
$mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=8 clean verify
Tests run: 76, Failures: 0, Errors: 0, Skipped: 0
Tests run: 441, Failures: 0, Errors: 0, Skipped: 33
Tests run: 206, Failures: 0, Errors: 0, Skipped: 24
---
.../fs/azurebfs/extensions/SASTokenProvider.java | 2 +-
.../ITestAzureBlobFileSystemDelegationSAS.java | 16 +++++++++++
.../extensions/MockDelegationSASTokenProvider.java | 4 +--
.../fs/azurebfs/utils/DelegationSASGenerator.java | 32 ++++++++++------------
.../hadoop/fs/azurebfs/utils/SASGenerator.java | 8 ++++--
.../fs/azurebfs/utils/ServiceSASGenerator.java | 6 ++--
6 files changed, 42 insertions(+), 26 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java
index 2cd44f1..a2cd292 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.security.AccessControlException;
public interface SASTokenProvider {
String CHECK_ACCESS_OPERATION = "check-access";
+ String CREATE_DIRECTORY_OPERATION = "create-directory";
String CREATE_FILE_OPERATION = "create-file";
String DELETE_OPERATION = "delete";
String DELETE_RECURSIVE_OPERATION = "delete-recursive";
@@ -40,7 +41,6 @@ public interface SASTokenProvider {
String GET_STATUS_OPERATION = "get-status";
String GET_PROPERTIES_OPERATION = "get-properties";
String LIST_OPERATION = "list";
- String CREATE_DIRECTORY_OPERATION = "create-directory";
String READ_OPERATION = "read";
String RENAME_SOURCE_OPERATION = "rename-source";
String RENAME_DESTINATION_OPERATION = "rename-destination";
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
index 07b5804..c2c691e 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.UUID;
+import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
import org.slf4j.Logger;
@@ -365,4 +366,19 @@ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrati
}
assertEquals(0, count);
}
+
+ @Test
+ // Test filesystem operations getXAttr and setXAttr
+ public void testProperties() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path reqPath = new Path(UUID.randomUUID().toString());
+
+ fs.create(reqPath).close();
+
+ final String propertyName = "user.mime_type";
+ final byte[] propertyValue = "text/plain".getBytes("utf-8");
+ fs.setXAttr(reqPath, propertyName, propertyValue);
+
+ assertArrayEquals(propertyValue, fs.getXAttr(reqPath, propertyName));
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java
index fa50bef..121256c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java
@@ -55,8 +55,8 @@ public class MockDelegationSASTokenProvider implements SASTokenProvider {
String appSecret = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET);
String sktid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID);
String skoid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID);
- String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minusSeconds(SASGenerator.FIVE_MINUTES));
- String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plusSeconds(SASGenerator.ONE_DAY));
+ String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minus(SASGenerator.FIVE_MINUTES));
+ String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plus(SASGenerator.ONE_DAY));
String skv = SASGenerator.AuthenticationVersion.Dec19.toString();
byte[] key = getUserDelegationKey(accountName, appID, appSecret, sktid, skt, ske, skv);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java
index f84ed6a..6f2209a 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java
@@ -29,12 +29,12 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsUriQueryBuilder;
* Test Delegation SAS generator.
*/
public class DelegationSASGenerator extends SASGenerator {
- private String skoid;
- private String sktid;
- private String skt;
- private String ske;
+ private final String skoid;
+ private final String sktid;
+ private final String skt;
+ private final String ske;
private final String sks = "b";
- private String skv;
+ private final String skv;
public DelegationSASGenerator(byte[] userDelegationKey, String skoid, String sktid, String skt, String ske, String skv) {
super(userDelegationKey);
@@ -48,20 +48,18 @@ public class DelegationSASGenerator extends SASGenerator {
public String getDelegationSAS(String accountName, String containerName, String path, String operation,
String saoid, String suoid, String scid) {
- final String sv = AuthenticationVersion.Dec19.toString();
- final String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES));
- final String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY));
+ final String sv = AuthenticationVersion.Feb20.toString();
+ final String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES));
+ final String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY));
String sr = "b";
String sdd = null;
- String sp = null;
+ String sp;
switch (operation) {
- case SASTokenProvider.CHECK_ACCESS_OPERATION:
- sp = "e";
- break;
- case SASTokenProvider.WRITE_OPERATION:
case SASTokenProvider.CREATE_FILE_OPERATION:
case SASTokenProvider.CREATE_DIRECTORY_OPERATION:
+ case SASTokenProvider.WRITE_OPERATION:
+ case SASTokenProvider.SET_PROPERTIES_OPERATION:
sp = "w";
break;
case SASTokenProvider.DELETE_OPERATION:
@@ -72,6 +70,7 @@ public class DelegationSASGenerator extends SASGenerator {
sr = "d";
sdd = Integer.toString(StringUtils.countMatches(path, "/"));
break;
+ case SASTokenProvider.CHECK_ACCESS_OPERATION:
case SASTokenProvider.GET_ACL_OPERATION:
case SASTokenProvider.GET_STATUS_OPERATION:
sp = "e";
@@ -79,6 +78,7 @@ public class DelegationSASGenerator extends SASGenerator {
case SASTokenProvider.LIST_OPERATION:
sp = "l";
break;
+ case SASTokenProvider.GET_PROPERTIES_OPERATION:
case SASTokenProvider.READ_OPERATION:
sp = "r";
break;
@@ -87,14 +87,12 @@ public class DelegationSASGenerator extends SASGenerator {
sp = "m";
break;
case SASTokenProvider.SET_ACL_OPERATION:
+ case SASTokenProvider.SET_PERMISSION_OPERATION:
sp = "p";
break;
case SASTokenProvider.SET_OWNER_OPERATION:
sp = "o";
break;
- case SASTokenProvider.SET_PERMISSION_OPERATION:
- sp = "p";
- break;
default:
throw new IllegalArgumentException(operation);
}
@@ -146,7 +144,7 @@ public class DelegationSASGenerator extends SASGenerator {
sb.append(accountName);
sb.append("/");
sb.append(containerName);
- if (path != null && sr != "c") {
+ if (path != null && !sr.equals("c")) {
sb.append(path);
}
sb.append("\n");
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java
index 34d504a..2e9289d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.azurebfs.utils;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.time.format.DateTimeFormatter;
import java.time.ZoneId;
import java.util.Locale;
@@ -35,7 +36,8 @@ public abstract class SASGenerator {
public enum AuthenticationVersion {
Nov18("2018-11-09"),
- Dec19("2019-12-12");
+ Dec19("2019-12-12"),
+ Feb20("2020-02-10");
private final String ver;
@@ -50,8 +52,8 @@ public abstract class SASGenerator {
}
protected static final Logger LOG = LoggerFactory.getLogger(SASGenerator.class);
- public static final int FIVE_MINUTES = 5 * 60;
- public static final int ONE_DAY = 24 * 60 * 60;
+ public static final Duration FIVE_MINUTES = Duration.ofMinutes(5);
+ public static final Duration ONE_DAY = Duration.ofDays(1);
public static final DateTimeFormatter ISO_8601_FORMATTER =
DateTimeFormatter
.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT)
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java
index a76681b..3d8496e 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java
@@ -40,8 +40,8 @@ public class ServiceSASGenerator extends SASGenerator {
String sp = "rcwdl";
String sv = AuthenticationVersion.Nov18.toString();
String sr = "c";
- String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES));
- String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY));
+ String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES));
+ String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY));
String signature = computeSignatureForSAS(sp, st, se, sv, "c",
accountName, containerName, null);
@@ -71,7 +71,7 @@ public class ServiceSASGenerator extends SASGenerator {
sb.append(accountName);
sb.append("/");
sb.append(containerName);
- if (path != null && sr != "c") {
+ if (path != null && !sr.equals("c")) {
//sb.append("/");
sb.append(path);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org