You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2016/11/22 22:05:38 UTC
hadoop git commit: HADOOP-13050. Upgrade to AWS SDK 10.11+.
Contributed by Chris Nauroth and Steve Loughran
Repository: hadoop
Updated Branches:
refs/heads/trunk 3a09e5970 -> af791b774
HADOOP-13050. Upgrade to AWS SDK 10.11+. Contributed by Chris Nauroth and Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af791b77
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af791b77
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af791b77
Branch: refs/heads/trunk
Commit: af791b774bdc3af26ab99487ffea7246c6788a9c
Parents: 3a09e59
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Nov 21 17:29:18 2016 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Nov 22 14:00:35 2016 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 7 ++++++-
hadoop-tools/hadoop-aws/pom.xml | 4 ++++
.../main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 6 ++++--
.../apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java | 3 ++-
.../java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java | 5 +++--
.../apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 3 +++
6 files changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 95fe839..112fa18 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -126,7 +126,7 @@
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.10.6</aws-java-sdk.version>
+ <aws-java-sdk.version>1.11.45</aws-java-sdk.version>
</properties>
<dependencyManagement>
@@ -876,6 +876,11 @@
<version>${jackson2.version}</version>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-cbor</artifactId>
+ <version>${jackson2.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.8.5</version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index e273428..1407661 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -345,6 +345,10 @@
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-cbor</artifactId>
+ </dependency>
+ <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index 0a4dd02..871322d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -180,7 +180,7 @@ interface S3ClientFactory {
userAgent = userAgentPrefix + ", " + userAgent;
}
LOG.debug("Using User-Agent: {}", userAgent);
- awsConf.setUserAgent(userAgent);
+ awsConf.setUserAgentPrefix(userAgent);
}
/**
@@ -225,7 +225,9 @@ interface S3ClientFactory {
final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
if (pathStyleAccess) {
LOG.debug("Enabling path style access!");
- s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
+ s3.setS3ClientOptions(S3ClientOptions.builder()
+ .setPathStyleAccess(true)
+ .build());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
index 819d9d8..d7f3411 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
@@ -110,7 +110,8 @@ public class ITestS3AAWSCredentialsProvider {
@SuppressWarnings("unused")
public GoodCredentialsProvider(URI name, Configuration conf) {
super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
- conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider());
+ conf.get(SECRET_KEY)),
+ InstanceProfileCredentialsProvider.getInstance());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 9163b15..f7d77f4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -399,7 +399,8 @@ public class ITestS3AConfiguration {
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
- assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent());
+ assertEquals("Hadoop " + VersionInfo.getVersion(),
+ awsConf.getUserAgentPrefix());
}
@Test
@@ -413,7 +414,7 @@ public class ITestS3AConfiguration {
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
- awsConf.getUserAgent());
+ awsConf.getUserAgentPrefix());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af791b77/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
index 8421dad..d8ae313 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
@@ -117,6 +117,9 @@ public class ITestS3AMiniYarnCluster extends AbstractS3ATestBase {
Map<String, Integer> result = new HashMap<>();
for (String line : outputAsStr.split("\n")) {
String[] tokens = line.split("\t");
+ assertTrue("Not enough tokens in in string \" "+ line
+ + "\" from output \"" + outputAsStr + "\"",
+ tokens.length > 1);
result.put(tokens[0], Integer.parseInt(tokens[1]));
}
return result;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org