You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/06/03 01:47:56 UTC
[24/50] [abbrv] hadoop git commit: HADOOP-11959. WASB should
configure client side socket timeout in storage client blob request options.
Contributed by Ivan Mitic.
HADOOP-11959. WASB should configure client side socket timeout in storage client blob request options. Contributed by Ivan Mitic.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44f4df57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44f4df57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44f4df57
Branch: refs/heads/YARN-2928
Commit: 44f4df57bfe7a078894cc4c05b07c0e3a292412f
Parents: 4b4aeba
Author: cnauroth <cn...@apache.org>
Authored: Thu May 28 12:31:06 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-project/pom.xml | 2 +-
.../hadoop/fs/azure/AzureNativeFileSystemStore.java | 13 ++-----------
.../org/apache/hadoop/fs/azure/StorageInterface.java | 6 +++---
.../apache/hadoop/fs/azure/StorageInterfaceImpl.java | 4 ++--
.../apache/hadoop/fs/azure/MockStorageInterface.java | 4 ++--
.../fs/azure/TestAzureFileSystemErrorConditions.java | 1 +
.../apache/hadoop/fs/azure/TestBlobDataValidation.java | 1 +
8 files changed, 15 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6aa224c..aba9087 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -791,6 +791,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11930. test-patch in offline mode should tell maven to be in
offline mode (Sean Busbey via aw)
+ HADOOP-11959. WASB should configure client side socket timeout in storage
+ client blob request options. (Ivan Mitic via cnauroth)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 94e1f5e..a5d4ed8 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -942,7 +942,7 @@
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
- <version>2.0.0</version>
+ <version>2.2.0</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 3267d8b..69bda06 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -2434,15 +2434,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
//
CloudBlobWrapper dstBlob = getBlobReference(dstKey);
- // TODO: Remove at the time when we move to Azure Java SDK 1.2+.
- // This is the workaround provided by Azure Java SDK team to
- // mitigate the issue with un-encoded x-ms-copy-source HTTP
- // request header. Azure sdk version before 1.2+ does not encode this
- // header what causes all URIs that have special (category "other")
- // characters in the URI not to work with startCopyFromBlob when
- // specified as source (requests fail with HTTP 403).
- URI srcUri = new URI(srcBlob.getUri().toASCIIString());
-
// Rename the source blob to the destination blob by copying it to
// the destination blob then deleting it.
//
@@ -2451,7 +2442,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// a more intensive exponential retry policy when the cluster is getting
// throttled.
try {
- dstBlob.startCopyFromBlob(srcUri, null, getInstrumentedContext());
+ dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext());
} catch (StorageException se) {
if (se.getErrorCode().equals(
StorageErrorCode.SERVER_BUSY.toString())) {
@@ -2475,7 +2466,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
options.setRetryPolicyFactory(new RetryExponentialRetry(
copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff,
copyBlobMaxRetries));
- dstBlob.startCopyFromBlob(srcUri, options, getInstrumentedContext());
+ dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext());
} else {
throw se;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
index e89151d..ce5f749 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
@@ -381,8 +381,8 @@ abstract class StorageInterface {
* Copies an existing blob's contents, properties, and metadata to this instance of the <code>CloudBlob</code>
* class, using the specified operation context.
*
- * @param source
- * A <code>java.net.URI</code> The URI of a source blob.
+ * @param sourceBlob
+ * A <code>CloudBlob</code> object that represents the source blob to copy.
* @param options
* A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
* <code>null</code> will use the default request options from the associated service client (
@@ -397,7 +397,7 @@ abstract class StorageInterface {
* @throws URISyntaxException
*
*/
- public abstract void startCopyFromBlob(URI source,
+ public abstract void startCopyFromBlob(CloudBlobWrapper sourceBlob,
BlobRequestOptions options, OperationContext opContext)
throws StorageException, URISyntaxException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
index 90d4d88..382ff66 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
@@ -393,10 +393,10 @@ class StorageInterfaceImpl extends StorageInterface {
}
@Override
- public void startCopyFromBlob(URI source, BlobRequestOptions options,
+ public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
OperationContext opContext)
throws StorageException, URISyntaxException {
- getBlob().startCopyFromBlob(source,
+ getBlob().startCopyFromBlob(((CloudBlobWrapperImpl)sourceBlob).blob,
null, null, options, opContext);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
index cde0e38..9f84f4b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
@@ -429,9 +429,9 @@ public class MockStorageInterface extends StorageInterface {
}
@Override
- public void startCopyFromBlob(URI source, BlobRequestOptions options,
+ public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
OperationContext opContext) throws StorageException, URISyntaxException {
- backingStore.copy(convertUriToDecodedString(source), convertUriToDecodedString(uri));
+ backingStore.copy(convertUriToDecodedString(sourceBlob.getUri()), convertUriToDecodedString(uri));
//TODO: set the backingStore.properties.CopyState and
// update azureNativeFileSystemStore.waitForCopyToComplete
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
index ace57dc..810bcf7 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
@@ -205,6 +205,7 @@ public class TestAzureFileSystemErrorConditions {
@Override
public boolean isTargetConnection(HttpURLConnection connection) {
return connection.getRequestMethod().equals("PUT")
+ && connection.getURL().getQuery() != null
&& connection.getURL().getQuery().contains("blocklist");
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
index 9237ade..c40b7b5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
@@ -191,6 +191,7 @@ public class TestBlobDataValidation {
private static boolean isPutBlock(HttpURLConnection connection) {
return connection.getRequestMethod().equals("PUT")
+ && connection.getURL().getQuery() != null
&& connection.getURL().getQuery().contains("blockid");
}