You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2022/10/10 12:35:32 UTC
[hadoop] branch branch-3.3.5 updated (cc5344e80fa -> 2aa77a75f9e)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch branch-3.3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from cc5344e80fa HADOOP-18468: Upgrade jettison to 1.5.1 to fix CVE-2022-40149 (#4937)
new cdbfc7abd39 HADOOP-18480. Upgrade aws sdk to 1.12.316 (#4972)
new 2aa77a75f9e HADOOP-18460. checkIfVectoredIOStopped before populating the buffers (#4986)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
LICENSE-binary | 2 +-
hadoop-project/pom.xml | 2 +-
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 43 ++++++++++++++--------
3 files changed, 30 insertions(+), 17 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HADOOP-18460. checkIfVectoredIOStopped before populating the buffers (#4986)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 2aa77a75f9ee59bdff0d03fc1eb65eaab72e353b
Author: Mukund Thakur <mt...@cloudera.com>
AuthorDate: Mon Oct 10 15:47:45 2022 +0530
HADOOP-18460. checkIfVectoredIOStopped before populating the buffers (#4986)
Contributed by Mukund Thakur
---
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 43 ++++++++++++++--------
1 file changed, 28 insertions(+), 15 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index 39d41f5ffd2..be5b1799b35 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -910,21 +910,15 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
private void readCombinedRangeAndUpdateChildren(CombinedFileRange combinedFileRange,
IntFunction<ByteBuffer> allocate) {
LOG.debug("Start reading combined range {} from path {} ", combinedFileRange, pathStr);
- // This reference is must be kept till all buffers are populated as this is a
+ // This reference must be kept till all buffers are populated as this is a
// finalizable object which closes the internal stream when gc triggers.
S3Object objectRange = null;
S3ObjectInputStream objectContent = null;
try {
- checkIfVectoredIOStopped();
- final String operationName = "readCombinedFileRange";
- objectRange = getS3Object(operationName,
+ objectRange = getS3ObjectAndValidateNotNull("readCombinedFileRange",
combinedFileRange.getOffset(),
combinedFileRange.getLength());
objectContent = objectRange.getObjectContent();
- if (objectContent == null) {
- throw new PathIOException(uri,
- "Null IO stream received during " + operationName);
- }
populateChildBuffers(combinedFileRange, objectContent, allocate);
} catch (Exception ex) {
LOG.debug("Exception while reading a range {} from path {} ", combinedFileRange, pathStr, ex);
@@ -1019,19 +1013,15 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
private void readSingleRange(FileRange range, ByteBuffer buffer) {
LOG.debug("Start reading range {} from path {} ", range, pathStr);
+ // This reference must be kept till all buffers are populated as this is a
+ // finalizable object which closes the internal stream when gc triggers.
S3Object objectRange = null;
S3ObjectInputStream objectContent = null;
try {
- checkIfVectoredIOStopped();
long position = range.getOffset();
int length = range.getLength();
- final String operationName = "readRange";
- objectRange = getS3Object(operationName, position, length);
+ objectRange = getS3ObjectAndValidateNotNull("readSingleRange", position, length);
objectContent = objectRange.getObjectContent();
- if (objectContent == null) {
- throw new PathIOException(uri,
- "Null IO stream received during " + operationName);
- }
populateBuffer(length, buffer, objectContent);
range.getData().complete(buffer);
} catch (Exception ex) {
@@ -1043,6 +1033,29 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
LOG.debug("Finished reading range {} from path {} ", range, pathStr);
}
+ /**
+ * Get the s3 object for S3 server for a specified range.
+ * Also checks if the vectored io operation has been stopped before and after
+ * the http get request such that we don't waste time populating the buffers.
+ * @param operationName name of the operation for which get object on S3 is called.
+ * @param position position of the object to be read from S3.
+ * @param length length from position of the object to be read from S3.
+ * @return result s3 object.
+ * @throws IOException exception if any.
+ */
+ private S3Object getS3ObjectAndValidateNotNull(final String operationName,
+ final long position,
+ final int length) throws IOException {
+ checkIfVectoredIOStopped();
+ S3Object objectRange = getS3Object(operationName, position, length);
+ if (objectRange.getObjectContent() == null) {
+ throw new PathIOException(uri,
+ "Null IO stream received during " + operationName);
+ }
+ checkIfVectoredIOStopped();
+ return objectRange;
+ }
+
/**
* Populates the buffer with data from objectContent
* till length. Handles both direct and heap byte buffers.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HADOOP-18480. Upgrade aws sdk to 1.12.316 (#4972)
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit cdbfc7abd392da5f05e20c5549fc55450bae366b
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Mon Oct 10 10:23:50 2022 +0100
HADOOP-18480. Upgrade aws sdk to 1.12.316 (#4972)
Contributed by Steve Loughran
---
LICENSE-binary | 2 +-
hadoop-project/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 3ff026a0d6f..ba33689a8fa 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -214,7 +214,7 @@ com.aliyun:aliyun-java-sdk-ecs:4.2.0
com.aliyun:aliyun-java-sdk-ram:3.0.0
com.aliyun:aliyun-java-sdk-sts:3.0.0
com.aliyun.oss:aliyun-sdk-oss:3.13.0
-com.amazonaws:aws-java-sdk-bundle:1.12.262
+com.amazonaws:aws-java-sdk-bundle:1.12.316
com.cedarsoftware:java-util:1.9.0
com.cedarsoftware:json-io:2.5.1
com.fasterxml.jackson.core:jackson-annotations:2.12.7
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8c3a4b940cb..4bb8f9d7c3e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -184,7 +184,7 @@
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.12.262</aws-java-sdk.version>
+ <aws-java-sdk.version>1.12.316</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org