You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2019/02/28 19:42:55 UTC
[hadoop] 01/02: HADOOP-16109. Parquet reading S3AFileSystem causes
EOF
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch s3/HADOOP-16109-parquet-eof-s3a-seek
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 3e7e4b6b0b14371819e2064317e88d77dd4d1785
Author: Steve Loughran <st...@apache.org>
AuthorDate: Thu Feb 28 18:53:52 2019 +0000
HADOOP-16109. Parquet reading S3AFileSystem causes EOF
This looks like a classic off-by-one error: the read-to-end of stream would take place even if the result was an depleated stream.
Change-Id: Ie24d51a099e538ad1794a41e662db81991c44b68
---
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 4 ++-
.../contract/s3a/ITestS3AContractRandomSeek.java | 33 ++++++++++++++++++++++
.../fs/contract/s3a/ITestS3AContractSeek.java | 32 +++++++++++++++++++++
3 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index ccc86d0..35dd834 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -261,7 +261,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
long forwardSeekLimit = Math.min(remainingInCurrentRequest,
forwardSeekRange);
boolean skipForward = remainingInCurrentRequest > 0
- && diff <= forwardSeekLimit;
+ && diff < forwardSeekLimit;
if (skipForward) {
// the forward seek range is within the limits
LOG.debug("Forward seek on {}, of {} bytes", uri, diff);
@@ -275,6 +275,8 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
if (pos == targetPos) {
// all is well
+ LOG.debug("Now at {}: bytes remaining in current request: {}",
+ pos, remainingInCurrentRequest());
return;
} else {
// log a warning; continue to attempt to re-open
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRandomSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRandomSeek.java
new file mode 100644
index 0000000..e1ed976
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRandomSeek.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.Constants;
+
+public class ITestS3AContractRandomSeek extends ITestS3AContractSeek {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.setInt(Constants.READAHEAD_RANGE, READAHEAD);
+ conf.set(Constants.INPUT_FADVISE, Constants.INPUT_FADV_RANDOM);
+ return conf;
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
index 379ace8..d968e54 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
@@ -18,9 +18,15 @@
package org.apache.hadoop.fs.contract.s3a;
+import org.junit.Test;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
@@ -29,6 +35,8 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
*/
public class ITestS3AContractSeek extends AbstractContractSeekTest {
+ protected static final int READAHEAD = 1024;
+
/**
* Create a configuration, possibly patching in S3Guard options.
* @return a configuration
@@ -45,4 +53,28 @@ public class ITestS3AContractSeek extends AbstractContractSeekTest {
protected AbstractFSContract createContract(Configuration conf) {
return new S3AContract(conf);
}
+
+ /**
+ * Test for HADOOP-16109: Parquet reading S3AFileSystem causes EOF.
+ */
+ @Test
+ public void testReadPastReadahead() throws Throwable {
+ Path path = path("testReadPastReadahead");
+ byte[] dataset = ContractTestUtils.dataset(READAHEAD * 2, 'a', 32);
+ FileSystem fs = getFileSystem();
+ ContractTestUtils.writeDataset(fs, path, dataset, dataset.length, READAHEAD,
+ true);
+ // forward seek reading across readahead boundary
+ try (FSDataInputStream in = fs.open(path)) {
+ final byte[] temp = new byte[5];
+ in.readByte();
+ in.readFully(1023, temp); // <-- works
+ }
+ // forward seek reading from end of readahead boundary
+ try (FSDataInputStream in = fs.open(path)) {
+ final byte[] temp = new byte[5];
+ in.readByte();
+ in.readFully(1024, temp); // <-- throws EOFException
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org