You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/12/05 21:56:05 UTC
[22/50] [abbrv] hadoop git commit: HADOOP-15957. WASB: Add asterisk
wildcard support for PageBlobDirSet.
HADOOP-15957. WASB: Add asterisk wildcard support for PageBlobDirSet.
Contributed by Da Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ccb640a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ccb640a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ccb640a
Branch: refs/heads/HDFS-12943
Commit: 7ccb640a66bb5bb9f657a2db85bcc8ae0ded4892
Parents: c9bfca2
Author: Da Zhou <da...@microsoft.com>
Authored: Fri Nov 30 10:12:41 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 30 10:12:41 2018 +0000
----------------------------------------------------------------------
.../fs/azure/AzureNativeFileSystemStore.java | 62 ++++++-
.../azure/ITestNativeAzureFileSystemLive.java | 4 +-
.../fs/azure/TestKeyPageBlobDirectories.java | 170 +++++++++++++++++++
3 files changed, 231 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ccb640a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index d2f9ca6..7bf3420 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -44,6 +44,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobContainerWrapper;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobDirectoryWrapper;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
@@ -241,6 +242,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private static final String HTTP_SCHEME = "http";
private static final String HTTPS_SCHEME = "https";
private static final String WASB_AUTHORITY_DELIMITER = "@";
+ private static final char ASTERISK_SYMBOL = '*';
private static final String AZURE_ROOT_CONTAINER = "$root";
private static final int DEFAULT_CONCURRENT_WRITES = 8;
@@ -1169,7 +1171,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
for (String currentDir : rawDirs) {
String myDir;
try {
- myDir = verifyAndConvertToStandardFormat(currentDir);
+ myDir = verifyAndConvertToStandardFormat(currentDir.trim());
} catch (URISyntaxException ex) {
throw new AzureException(String.format(
"The directory %s specified in the configuration entry %s is not"
@@ -1214,7 +1216,12 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
public boolean isKeyForDirectorySet(String key, Set<String> dirSet) {
String defaultFS = FileSystem.getDefaultUri(sessionConfiguration).toString();
for (String dir : dirSet) {
- if (dir.isEmpty() || key.startsWith(dir + "/")) {
+ if (dir.isEmpty()) {
+ // dir is root
+ return true;
+ }
+
+ if (matchAsteriskPattern(key, dir)) {
return true;
}
@@ -1227,7 +1234,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Concatenate the default file system prefix with the relative
// page blob directory path.
//
- if (key.startsWith(trim(defaultFS, "/") + "/" + dir + "/")){
+ String dirWithPrefix = trim(defaultFS, "/") + "/" + dir;
+ if (matchAsteriskPattern(key, dirWithPrefix)) {
return true;
}
}
@@ -1238,6 +1246,54 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
return false;
}
+ private boolean matchAsteriskPattern(String pathName, String pattern) {
+ if (pathName == null || pathName.length() == 0) {
+ return false;
+ }
+
+ int pathIndex = 0;
+ int patternIndex = 0;
+
+ while (pathIndex < pathName.length() && patternIndex < pattern.length()) {
+ char charToMatch = pattern.charAt(patternIndex);
+
+ // normal char:
+ if (charToMatch != ASTERISK_SYMBOL) {
+ if (charToMatch != pathName.charAt(pathIndex)) {
+ return false;
+ }
+ pathIndex++;
+ patternIndex++;
+ continue;
+ }
+
+ // ASTERISK_SYMBOL
+ // 1. * is used in path name: *a/b,a*/b, a/*b, a/b*
+ if (patternIndex > 0 && pattern.charAt(patternIndex - 1) != Path.SEPARATOR_CHAR
+ || patternIndex + 1 < pattern.length() && pattern.charAt(patternIndex + 1) != Path.SEPARATOR_CHAR) {
+ if (ASTERISK_SYMBOL != pathName.charAt(pathIndex)) {
+ return false;
+ }
+
+ pathIndex++;
+ patternIndex++;
+ continue;
+ }
+
+ // 2. * is used as wildcard: */a, a/*/b, a/*
+ patternIndex++;
+ // find next path separator
+ while (pathIndex < pathName.length() && pathName.charAt(pathIndex) != Path.SEPARATOR_CHAR) {
+ pathIndex++;
+ }
+ }
+
+ // Ensure it is not a file/dir which shares same prefix as pattern
+ // Eg: pattern: /A/B, pathName: /A/BBB should not match
+ return patternIndex == pattern.length()
+ && (pathIndex == pathName.length() || pathName.charAt(pathIndex) == Path.SEPARATOR_CHAR);
+ }
+
/**
* Returns the file block size. This is a fake value used for integration
* of the Azure store with Hadoop.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ccb640a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
index 9033674..f86af95 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
@@ -230,7 +230,7 @@ public class ITestNativeAzureFileSystemLive extends
// negative tests
String[] negativeKeys = { "", "/", "bar", "bar/", "bar/pageBlobs", "bar/pageBlobs/foo",
- "bar/pageBlobs/foo/", "/pageBlobs/", "/pageBlobs", "pageBlobs", "pageBlobsxyz/" };
+ "bar/pageBlobs/foo/", "/pageBlobs/", "/pageBlobs", "pageBlobsxyz/" };
for (String s : negativeKeys) {
assertFalse(store.isPageBlobKey(s));
assertFalse(store.isPageBlobKey(uriPrefix + s));
@@ -262,7 +262,7 @@ public class ITestNativeAzureFileSystemLive extends
// negative tests
String[] negativeKeys = { "", "/", "bar", "bar/", "bar/hbase",
- "bar/hbase/foo", "bar/hbase/foo/", "/hbase/", "/hbase", "hbase",
+ "bar/hbase/foo", "bar/hbase/foo/", "/hbase/", "/hbase",
"hbasexyz/", "foo/atomicRenameDir1/"};
for (String s : negativeKeys) {
assertFalse(store.isAtomicRenameKey(s));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ccb640a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestKeyPageBlobDirectories.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestKeyPageBlobDirectories.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestKeyPageBlobDirectories.java
new file mode 100644
index 0000000..7cc4878
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestKeyPageBlobDirectories.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.junit.Test;
+
+/**
+ * Test config property KEY_PAGE_BLOB_DIRECTORIES.
+ */
+public class TestKeyPageBlobDirectories extends AbstractWasbTestBase{
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ return AzureBlobStorageTestAccount.create();
+ }
+
+ public void expectPageBlobKey(boolean expectedOutcome, AzureNativeFileSystemStore store, String path) {
+ assertEquals("Unexpected result for isPageBlobKey(" + path + ")",
+ expectedOutcome, store.isPageBlobKey(path));
+
+ }
+
+ @Test
+ public void testKeySetWithoutAsterisk() throws Exception {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ String dirList = "/service/WALs,/data/mypageblobfiles";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ expectPageBlobKey(false, store, "/");
+ expectPageBlobKey(false, store, "service");
+
+ expectPageBlobKey(false, store, "service/dir/recovered.edits");
+ expectPageBlobKey(true, store, "service/WALs/recovered.edits");
+
+ expectPageBlobKey(false, store, "data/dir/recovered.txt");
+ expectPageBlobKey(true, store, "data/mypageblobfiles/recovered.txt");
+ }
+
+ @Test
+ public void testKeySetWithAsterisk() throws Exception {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ String dirList = "/service/*/*/*/recovered.edits,/*/recovered.edits,/*/*/*/WALs, /*/*/oldWALs/*/*";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ expectPageBlobKey(false, store, "/");
+ expectPageBlobKey(false, store, "service");
+
+ expectPageBlobKey(false, store, "service/dir/recovered.edits");
+ expectPageBlobKey(true, store, "service/dir1/dir2/dir3/recovered.edits");
+
+ expectPageBlobKey(false, store, "data/dir/recovered.edits");
+ expectPageBlobKey(true, store, "data/recovered.edits");
+
+ expectPageBlobKey(false, store, "dir1/dir2/WALs/data");
+ expectPageBlobKey(true, store, "dir1/dir2/dir3/WALs/data1");
+ expectPageBlobKey(true, store, "dir1/dir2/dir3/WALs/data2");
+
+ expectPageBlobKey(false, store, "dir1/oldWALs/data");
+ expectPageBlobKey(false, store, "dir1/dir2/oldWALs/data");
+ expectPageBlobKey(true, store, "dir1/dir2/oldWALs/dir3/dir4/data");
+ }
+
+
+
+ @Test
+ public void testKeySetUsingFullName() throws Exception {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ String dirList = "/service/WALs,/data/mypageblobfiles,/*/*/WALs,/*/*/recover.edits";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ final String defaultFS = FileSystem.getDefaultUri(conf).toString();
+
+ expectPageBlobKey(false, store, defaultFS + "service/recover.edits");
+ expectPageBlobKey(true, store, defaultFS + "service/WALs/recover.edits");
+
+ expectPageBlobKey(false, store, defaultFS + "data/mismatch/mypageblobfiles/data");
+ expectPageBlobKey(true, store, defaultFS + "data/mypageblobfiles/data");
+
+ expectPageBlobKey(false, store, defaultFS + "dir1/dir2/dir3/WALs/data");
+ expectPageBlobKey(true, store, defaultFS + "dir1/dir2/WALs/data");
+
+ expectPageBlobKey(false, store, defaultFS + "dir1/dir2/dir3/recover.edits");
+ expectPageBlobKey(true, store, defaultFS + "dir1/dir2/recover.edits");
+
+ }
+
+ @Test
+ public void testKeyContainsAsterisk() throws IOException {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ // Test dir name which contains *
+ String dirList = "/service/*/*/*/d*ir,/*/fi**le.data,/*/*/*/WALs*, /*/*/oldWALs";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ expectPageBlobKey(false, store, "/");
+ expectPageBlobKey(false, store, "service");
+
+ expectPageBlobKey(false, store, "service/d*ir/data");
+ expectPageBlobKey(true, store, "service/dir1/dir2/dir3/d*ir/data");
+
+ expectPageBlobKey(false, store, "dir/fi*le.data");
+ expectPageBlobKey(true, store, "dir/fi**le.data");
+
+ expectPageBlobKey(false, store, "dir1/dir2/WALs/data");
+ expectPageBlobKey(false, store, "dir1/dir2/dir3/WALs/data");
+ expectPageBlobKey(true, store, "dir1/dir2/dir3/WALs*/data1");
+ expectPageBlobKey(true, store, "dir1/dir2/dir3/WALs*/data2");
+
+ expectPageBlobKey(false, store, "dir1/oldWALs/data");
+ expectPageBlobKey(true, store, "dir1/dir2/oldWALs/data1");
+ expectPageBlobKey(true, store, "dir1/dir2/oldWALs/data2");
+ }
+
+ @Test
+ public void testKeyWithCommonPrefix() throws IOException {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ // Test dir name which contains *
+ String dirList = "/service/WALs,/*/*/WALs";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ expectPageBlobKey(false, store, "/");
+ expectPageBlobKey(false, store, "service");
+
+ expectPageBlobKey(false, store, "service/WALsssssss/dir");
+ expectPageBlobKey(true, store, "service/WALs/dir");
+
+ expectPageBlobKey(false, store, "service/dir/WALsss/data");
+ expectPageBlobKey(true, store, "service/dir/WALs/data");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org