You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2022/10/10 12:31:46 UTC
[hadoop] branch branch-3.3 updated (e360e7620c9 -> 77cb778a446)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from e360e7620c9 HADOOP-18468: Upgrade jettison to 1.5.1 to fix CVE-2022-40149 (#4937)
new 80525615e57 HADOOP-18480. Upgrade aws sdk to 1.12.316 (#4972)
new 77cb778a446 HADOOP-18460. checkIfVectoredIOStopped before populating the buffers (#4986)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
LICENSE-binary | 2 +-
hadoop-project/pom.xml | 2 +-
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 43 ++++++++++++++--------
3 files changed, 30 insertions(+), 17 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HADOOP-18480. Upgrade aws sdk to 1.12.316 (#4972)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 80525615e571d11dffdf14e68e2e1f1b689eeb86
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Mon Oct 10 10:23:50 2022 +0100
HADOOP-18480. Upgrade aws sdk to 1.12.316 (#4972)
Contributed by Steve Loughran
---
LICENSE-binary | 2 +-
hadoop-project/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 3ff026a0d6f..ba33689a8fa 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -214,7 +214,7 @@ com.aliyun:aliyun-java-sdk-ecs:4.2.0
com.aliyun:aliyun-java-sdk-ram:3.0.0
com.aliyun:aliyun-java-sdk-sts:3.0.0
com.aliyun.oss:aliyun-sdk-oss:3.13.0
-com.amazonaws:aws-java-sdk-bundle:1.12.262
+com.amazonaws:aws-java-sdk-bundle:1.12.316
com.cedarsoftware:java-util:1.9.0
com.cedarsoftware:json-io:2.5.1
com.fasterxml.jackson.core:jackson-annotations:2.12.7
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index adc82e4c5bf..8c7111855f5 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -184,7 +184,7 @@
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.12.262</aws-java-sdk.version>
+ <aws-java-sdk.version>1.12.316</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HADOOP-18460. checkIfVectoredIOStopped before populating the buffers (#4986)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 77cb778a4462a2da26663b431e9e179e9f6b3208
Author: Mukund Thakur <mt...@cloudera.com>
AuthorDate: Mon Oct 10 15:47:45 2022 +0530
HADOOP-18460. checkIfVectoredIOStopped before populating the buffers (#4986)
Contributed by Mukund Thakur
---
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 43 ++++++++++++++--------
1 file changed, 28 insertions(+), 15 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index 39d41f5ffd2..be5b1799b35 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -910,21 +910,15 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
private void readCombinedRangeAndUpdateChildren(CombinedFileRange combinedFileRange,
IntFunction<ByteBuffer> allocate) {
LOG.debug("Start reading combined range {} from path {} ", combinedFileRange, pathStr);
- // This reference is must be kept till all buffers are populated as this is a
+ // This reference must be kept till all buffers are populated as this is a
// finalizable object which closes the internal stream when gc triggers.
S3Object objectRange = null;
S3ObjectInputStream objectContent = null;
try {
- checkIfVectoredIOStopped();
- final String operationName = "readCombinedFileRange";
- objectRange = getS3Object(operationName,
+ objectRange = getS3ObjectAndValidateNotNull("readCombinedFileRange",
combinedFileRange.getOffset(),
combinedFileRange.getLength());
objectContent = objectRange.getObjectContent();
- if (objectContent == null) {
- throw new PathIOException(uri,
- "Null IO stream received during " + operationName);
- }
populateChildBuffers(combinedFileRange, objectContent, allocate);
} catch (Exception ex) {
LOG.debug("Exception while reading a range {} from path {} ", combinedFileRange, pathStr, ex);
@@ -1019,19 +1013,15 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
private void readSingleRange(FileRange range, ByteBuffer buffer) {
LOG.debug("Start reading range {} from path {} ", range, pathStr);
+ // This reference must be kept till all buffers are populated as this is a
+ // finalizable object which closes the internal stream when gc triggers.
S3Object objectRange = null;
S3ObjectInputStream objectContent = null;
try {
- checkIfVectoredIOStopped();
long position = range.getOffset();
int length = range.getLength();
- final String operationName = "readRange";
- objectRange = getS3Object(operationName, position, length);
+ objectRange = getS3ObjectAndValidateNotNull("readSingleRange", position, length);
objectContent = objectRange.getObjectContent();
- if (objectContent == null) {
- throw new PathIOException(uri,
- "Null IO stream received during " + operationName);
- }
populateBuffer(length, buffer, objectContent);
range.getData().complete(buffer);
} catch (Exception ex) {
@@ -1043,6 +1033,29 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
LOG.debug("Finished reading range {} from path {} ", range, pathStr);
}
+ /**
+ * Get the s3 object for S3 server for a specified range.
+ * Also checks if the vectored io operation has been stopped before and after
+ * the http get request such that we don't waste time populating the buffers.
+ * @param operationName name of the operation for which get object on S3 is called.
+ * @param position position of the object to be read from S3.
+ * @param length length from position of the object to be read from S3.
+ * @return result s3 object.
+ * @throws IOException exception if any.
+ */
+ private S3Object getS3ObjectAndValidateNotNull(final String operationName,
+ final long position,
+ final int length) throws IOException {
+ checkIfVectoredIOStopped();
+ S3Object objectRange = getS3Object(operationName, position, length);
+ if (objectRange.getObjectContent() == null) {
+ throw new PathIOException(uri,
+ "Null IO stream received during " + operationName);
+ }
+ checkIfVectoredIOStopped();
+ return objectRange;
+ }
+
/**
* Populates the buffer with data from objectContent
* till length. Handles both direct and heap byte buffers.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org