You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/10/30 19:00:31 UTC
[18/30] hadoop git commit: HADOOP-11685. StorageException complaining
" no lease ID" during HBase distributed log splitting. Contributed by Duo Xu.
HADOOP-11685. StorageException complaining " no lease ID" during HBase distributed log splitting. Contributed by Duo Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f7ecb0c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f7ecb0c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f7ecb0c
Branch: refs/heads/HDFS-8966
Commit: 1f7ecb0c84042783f9fcf3f77d7d889dc58c9ead
Parents: 73822de
Author: cnauroth <cn...@apache.org>
Authored: Tue Oct 27 22:56:22 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Tue Oct 27 22:56:22 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../fs/azure/AzureNativeFileSystemStore.java | 15 +++++++++-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 5 ++--
.../fs/azure/TestNativeAzureFileSystemLive.java | 29 +++++++++++++++++++-
4 files changed, 48 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f7ecb0c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 016fec8..25a3a60 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -919,6 +919,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12520. Use XInclude in hadoop-azure test configuration to isolate
Azure Storage account keys for service integration tests. (cnauroth)
+ HADOOP-11685. StorageException complaining " no lease ID" during HBase
+ distributed log splitting (Duo Xu via cnauroth)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f7ecb0c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 8a33742..6412714 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -1503,10 +1503,23 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
storePermissionStatus(blob, permissionStatus);
storeFolderAttribute(blob);
openOutputStream(blob).close();
- } catch (Exception e) {
+ } catch (StorageException e) {
// Caught exception while attempting upload. Re-throw as an Azure
// storage exception.
throw new AzureException(e);
+ } catch (URISyntaxException e) {
+ throw new AzureException(e);
+ } catch (IOException e) {
+ Throwable t = e.getCause();
+ if (t != null && t instanceof StorageException) {
+ StorageException se = (StorageException) t;
+ // If we got this exception, the blob should have already been created
+ if (!se.getErrorCode().equals("LeaseIdMissing")) {
+ throw new AzureException(e);
+ }
+ } else {
+ throw new AzureException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f7ecb0c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 9305b24..7c5a504 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -1870,8 +1870,9 @@ public class NativeAzureFileSystem extends FileSystem {
* @throws IOException
* If login fails in getCurrentUser
*/
- private PermissionStatus createPermissionStatus(FsPermission permission)
- throws IOException {
+ @VisibleForTesting
+ PermissionStatus createPermissionStatus(FsPermission permission)
+ throws IOException {
// Create the permission status for this file based on current user
return new PermissionStatus(
UserGroupInformation.getCurrentUser().getShortUserName(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f7ecb0c/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
index 985c0e9d..b033460 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
@@ -21,7 +21,8 @@ package org.apache.hadoop.fs.azure;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.Test;
/*
@@ -104,4 +105,30 @@ public class TestNativeAzureFileSystemLive extends
assertTrue(store.isAtomicRenameKey(uriPrefix + s));
}
}
+
+ /**
+ * Tests fs.mkdir() function to create a target blob while another thread
+ * is holding the lease on the blob. mkdir should not fail since the blob
+ * already exists.
+ * This is a scenario that would happen in HBase distributed log splitting.
+ * Multiple threads will try to create and update "recovered.edits" folder
+ * under the same path.
+ */
+ @Test
+ public void testMkdirOnExistingFolderWithLease() throws Exception {
+ SelfRenewingLease lease;
+ final String FILE_KEY = "folderWithLease";
+ // Create the folder
+ fs.mkdirs(new Path(FILE_KEY));
+ NativeAzureFileSystem nfs = (NativeAzureFileSystem) fs;
+ String fullKey = nfs.pathToKey(nfs.makeAbsolute(new Path(FILE_KEY)));
+ AzureNativeFileSystemStore store = nfs.getStore();
+ // Acquire the lease on the folder
+ lease = store.acquireLease(fullKey);
+ assertTrue(lease.getLeaseID() != null);
+ // Try to create the same folder
+ store.storeEmptyFolder(fullKey,
+ nfs.createPermissionStatus(FsPermission.getDirDefault()));
+ lease.free();
+ }
}