You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2019/03/18 10:00:53 UTC
[hadoop] branch branch-2.8 updated: HADOOP-16055. Upgrade AWS SDK
to 1.11.271 in branch-2.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.8 by this push:
new 6d0c160 HADOOP-16055. Upgrade AWS SDK to 1.11.271 in branch-2.
6d0c160 is described below
commit 6d0c160d45e4e7b97c0c273222cd6b552a323152
Author: Akira Ajisaka <aa...@apache.org>
AuthorDate: Mon Mar 18 09:55:19 2019 +0000
HADOOP-16055. Upgrade AWS SDK to 1.11.271 in branch-2.
Contains HADOOP-12705 Upgrade Jackson 2.2.3 to 2.7.8.
This change was required to address license compatibility issues with the JSON parser in the older AWS SDKs.
A consequence of this is that the version of Jackson 2 shipped is now 2.7.8.
Author: Akira Ajisaka <aa...@apache.org>
---
NOTICE.txt | 10 ++++------
hadoop-project/pom.xml | 17 +++++++++--------
hadoop-tools/hadoop-aws/pom.xml | 7 +------
.../java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 6 ++++--
.../hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java | 3 ++-
.../org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java | 5 +++--
6 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/NOTICE.txt b/NOTICE.txt
index 33c12ed..3b0784a 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -17,13 +17,11 @@ which has the following notices:
* This product includes software developed by IBM Corporation and others.
The binary distribution of this product bundles binaries of
-AWS Java SDK 1.10.6,
+AWS Java SDK 1.11.271,
which has the following notices:
- * This software includes third party software subject to the following
- copyrights: - XML parsing and utility functions from JetS3t - Copyright
- 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org -
- Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility
- functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
+ * This software includes third party software subject to the following copyrights:
+ - XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
+ - PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
The binary distribution of this product bundles binaries of
Gson 2.2.4,
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 4810166..6bcecd7 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -68,7 +68,7 @@
<!-- jackson versions -->
<jackson.version>1.9.13</jackson.version>
- <jackson2.version>2.2.3</jackson2.version>
+ <jackson2.version>2.7.8</jackson2.version>
<!-- ProtocolBuffer version, used to verify the protoc version and -->
<!-- define the protobuf JAR version -->
@@ -117,7 +117,7 @@
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.10.6</aws-java-sdk.version>
+ <aws-java-sdk.version>1.11.271</aws-java-sdk.version>
</properties>
<dependencyManagement>
@@ -684,13 +684,14 @@
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- <version>${aws-java-sdk.version}</version>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-sts</artifactId>
+ <artifactId>aws-java-sdk-bundle</artifactId>
<version>${aws-java-sdk.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.mina</groupId>
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index ad05f87..3065013 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -316,7 +316,7 @@
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
+ <artifactId>aws-java-sdk-bundle</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
@@ -336,11 +336,6 @@
<artifactId>joda-time</artifactId>
</dependency>
<dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-sts</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index 992c6ab..bc9d26a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -184,7 +184,7 @@ interface S3ClientFactory {
userAgent = userAgentPrefix + ", " + userAgent;
}
LOG.debug("Using User-Agent: {}", userAgent);
- awsConf.setUserAgent(userAgent);
+ awsConf.setUserAgentPrefix(userAgent);
}
/**
@@ -229,7 +229,9 @@ interface S3ClientFactory {
final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
if (pathStyleAccess) {
LOG.debug("Enabling path style access!");
- s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
+ s3.setS3ClientOptions(S3ClientOptions.builder()
+ .setPathStyleAccess(true)
+ .build());
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
index b2d410d..22c4f7e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
@@ -148,7 +148,8 @@ public class ITestS3AAWSCredentialsProvider {
@SuppressWarnings("unused")
public GoodCredentialsProvider(Configuration conf) {
super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
- conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider());
+ conf.get(SECRET_KEY)),
+ InstanceProfileCredentialsProvider.getInstance());
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 6625dc2..4580cc6 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -436,7 +436,8 @@ public class ITestS3AConfiguration {
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
- assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent());
+ assertEquals("Hadoop " + VersionInfo.getVersion(),
+ awsConf.getUserAgentPrefix());
}
@Test
@@ -450,7 +451,7 @@ public class ITestS3AConfiguration {
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
- awsConf.getUserAgent());
+ awsConf.getUserAgentPrefix());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org