You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2018/10/09 19:03:12 UTC
hadoop git commit: HADOOP-15825. ABFS: Enable some tests for
namespace not enabled account using OAuth. Contributed by Da Zhou.
Repository: hadoop
Updated Branches:
refs/heads/trunk a05bd1288 -> bd50fa956
HADOOP-15825. ABFS: Enable some tests for namespace not enabled account using OAuth.
Contributed by Da Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd50fa95
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd50fa95
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd50fa95
Branch: refs/heads/trunk
Commit: bd50fa956b1ca25bb2136977b98a6aa6895eff8b
Parents: a05bd12
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 9 20:02:12 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 9 20:02:12 2018 +0100
----------------------------------------------------------------------
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 3 ++-
.../azurebfs/ITestAzureBlobFileSystemBackCompat.java | 6 ++----
.../azurebfs/ITestAzureBlobFileSystemFileStatus.java | 12 ++++++------
.../fs/azurebfs/ITestAzureBlobFileSystemFinalize.java | 8 +++-----
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 5 -----
.../ITestAzureBlobFileSystemInitAndCreate.java | 3 +--
.../azurebfs/ITestAzureBlobFileSystemRandomRead.java | 12 +++++-------
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 14 ++++++++------
8 files changed, 27 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd50fa95/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index 6f794d0..382cd7f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -133,7 +133,8 @@ public abstract class AbstractAbfsIntegrationTest extends
//Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem.
createFileSystem();
- if (!isIPAddress && authType == AuthType.SharedKey) {
+ // Only live account without namespace support can run ABFS&WASB compatibility tests
+ if (!isIPAddress && !abfs.getIsNamespaceEnabled()) {
final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl()));
final AzureNativeFileSystemStore azureNativeFileSystemStore =
new AzureNativeFileSystemStore();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd50fa95/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
index 22d4990..5ac16b4 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
@@ -26,7 +26,6 @@ import com.microsoft.azure.storage.blob.CloudBlockBlob;
import org.junit.Assume;
import org.junit.Test;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@@ -38,14 +37,13 @@ public class ITestAzureBlobFileSystemBackCompat extends
public ITestAzureBlobFileSystemBackCompat() throws Exception {
super();
- Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
}
@Test
public void testBlobBackCompat() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- // test only valid for non-namespace enabled account
- Assume.assumeFalse(fs.getIsNamespaceEnabled());
+ Assume.assumeFalse("This test does not support namespace enabled account",
+ this.getFileSystem().getIsNamespaceEnabled());
String storageConnectionString = getBlobConnectionString();
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd50fa95/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
index 707a145..f514696 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.fs.azurebfs;
import java.io.IOException;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
-import org.junit.Ignore;
import org.junit.Test;
import org.apache.hadoop.fs.FileStatus;
@@ -37,6 +35,7 @@ public class ITestAzureBlobFileSystemFileStatus extends
private static final String DEFAULT_FILE_PERMISSION_VALUE = "640";
private static final String DEFAULT_DIR_PERMISSION_VALUE = "750";
private static final String DEFAULT_UMASK_VALUE = "027";
+ private static final String FULL_PERMISSION = "777";
private static final Path TEST_FILE = new Path("testFile");
private static final Path TEST_FOLDER = new Path("testDir");
@@ -54,7 +53,6 @@ public class ITestAzureBlobFileSystemFileStatus extends
assertEquals("root listing", 0, rootls.length);
}
- @Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.")
@Test
public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
@@ -69,13 +67,16 @@ public class ITestAzureBlobFileSystemFileStatus extends
String errorInStatus = "error in " + fileStatus + " from " + fs;
- // When running with Oauth, the owner and group info retrieved from server will be digit ids.
- if (this.getAuthType() != AuthType.OAuth && !fs.isSecureScheme()) {
+ if (!fs.getIsNamespaceEnabled()) {
assertEquals(errorInStatus + ": owner",
fs.getOwnerUser(), fileStatus.getOwner());
assertEquals(errorInStatus + ": group",
fs.getOwnerUserPrimaryGroup(), fileStatus.getGroup());
+ assertEquals(new FsPermission(FULL_PERMISSION), fileStatus.getPermission());
} else {
+ // When running with namespace enabled account,
+ // the owner and group info retrieved from server will be digit ids.
+ // hence skip the owner and group validation
if (isDir) {
assertEquals(errorInStatus + ": permission",
new FsPermission(DEFAULT_DIR_PERMISSION_VALUE), fileStatus.getPermission());
@@ -88,7 +89,6 @@ public class ITestAzureBlobFileSystemFileStatus extends
return fileStatus;
}
- @Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.")
@Test
public void testFolderStatusPermissionsAndOwnerAndGroup() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd50fa95/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java
index 9d17388..3c21525 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java
@@ -25,14 +25,13 @@ import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
/**
* Test finalize() method when "fs.abfs.impl.disable.cache" is enabled.
*/
public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{
static final String DISABLE_ABFS_CACHE_KEY = "fs.abfs.impl.disable.cache";
- static final String DISABLE_ABFSSS_CACHE_KEY = "fs.abfss.impl.disable.cache";
+ static final String DISABLE_ABFSS_CACHE_KEY = "fs.abfss.impl.disable.cache";
public ITestAzureBlobFileSystemFinalize() throws Exception {
super();
@@ -42,9 +41,8 @@ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{
public void testFinalize() throws Exception {
// Disable the cache for filesystem to make sure there is no reference.
Configuration rawConfig = this.getRawConfiguration();
- rawConfig.setBoolean(
- this.getAuthType() == AuthType.SharedKey ? DISABLE_ABFS_CACHE_KEY : DISABLE_ABFSSS_CACHE_KEY,
- true);
+ rawConfig.setBoolean(DISABLE_ABFS_CACHE_KEY, true);
+ rawConfig.setBoolean(DISABLE_ABFSS_CACHE_KEY, true);
AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd50fa95/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
index 23a1ab5..d60cae8 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
import org.hamcrest.core.IsEqual;
import org.hamcrest.core.IsNot;
-import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -42,8 +41,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
-
/**
* Test flush operation.
* This class cannot be run in parallel test mode--check comments in
@@ -221,8 +218,6 @@ public class ITestAzureBlobFileSystemFlush extends AbstractAbfsScaleTest {
}
private void testFlush(boolean flushEnabled) throws Exception {
- Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
-
final AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem();
// Simulate setting "fs.azure.enable.flush" to true or false
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd50fa95/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
index 5f08721..5c4b87b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
@@ -46,8 +46,7 @@ public class ITestAzureBlobFileSystemInitAndCreate extends
@Test (expected = FileNotFoundException.class)
public void ensureFilesystemWillNotBeCreatedIfCreationConfigIsNotSet() throws Exception {
- super.setup();
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = this.createFileSystem();
FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd50fa95/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
index 768f4bb..e5f64b5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -67,7 +66,6 @@ public class ITestAzureBlobFileSystemRandomRead extends
public ITestAzureBlobFileSystemRandomRead() throws Exception {
super();
- Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
}
@Test
@@ -98,6 +96,8 @@ public class ITestAzureBlobFileSystemRandomRead extends
*/
@Test
public void testRandomRead() throws Exception {
+ Assume.assumeFalse("This test does not support namespace enabled account",
+ this.getFileSystem().getIsNamespaceEnabled());
assumeHugeFileExists();
try (
FSDataInputStream inputStreamV1
@@ -413,6 +413,8 @@ public class ITestAzureBlobFileSystemRandomRead extends
@Test
public void testRandomReadPerformance() throws Exception {
+ Assume.assumeFalse("This test does not support namespace enabled account",
+ this.getFileSystem().getIsNamespaceEnabled());
createTestFile();
assumeHugeFileExists();
@@ -523,11 +525,7 @@ public class ITestAzureBlobFileSystemRandomRead extends
}
private void createTestFile() throws Exception {
- final AzureBlobFileSystem abFs = this.getFileSystem();
- // test only valid for non-namespace enabled account
- Assume.assumeFalse(abFs.getIsNamespaceEnabled());
- FileSystem fs = this.getWasbFileSystem();
-
+ final AzureBlobFileSystem fs = this.getFileSystem();
if (fs.exists(TEST_FILE_PATH)) {
FileStatus status = fs.getFileStatus(TEST_FILE_PATH);
if (status.getLen() >= TEST_FILE_SIZE) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd50fa95/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
index fdf9788..e8f845c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDeleted;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
@@ -51,7 +50,6 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
public ITestWasbAbfsCompatibility() throws Exception {
Assume.assumeFalse("Emulator is not supported", isIPAddress());
- Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
}
@Test
@@ -59,7 +57,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
// crate file using abfs
AzureBlobFileSystem fs = getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(fs.getIsNamespaceEnabled());
+ Assume.assumeFalse("Namespace enabled account does not support this test,",
+ fs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
@@ -93,7 +92,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
AzureBlobFileSystem abfs = getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(abfs.getIsNamespaceEnabled());
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ abfs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
@@ -132,7 +132,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
AzureBlobFileSystem abfs = getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(abfs.getIsNamespaceEnabled());
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ abfs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
@@ -166,7 +167,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
//create folders
AzureBlobFileSystem abfs = getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(abfs.getIsNamespaceEnabled());
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ abfs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org