You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2019/02/28 19:42:55 UTC

[hadoop] 01/02: HADOOP-16109. Parquet reading S3AFileSystem causes EOF

This is an automated email from the ASF dual-hosted git repository.

stevel pushed a commit to branch s3/HADOOP-16109-parquet-eof-s3a-seek
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 3e7e4b6b0b14371819e2064317e88d77dd4d1785
Author: Steve Loughran <st...@apache.org>
AuthorDate: Thu Feb 28 18:53:52 2019 +0000

    HADOOP-16109. Parquet reading S3AFileSystem causes EOF
    
    This looks like a classic off-by-one error: the read-to-end of stream would take place even if the result was an depleated stream.
    
    Change-Id: Ie24d51a099e538ad1794a41e662db81991c44b68
---
 .../org/apache/hadoop/fs/s3a/S3AInputStream.java   |  4 ++-
 .../contract/s3a/ITestS3AContractRandomSeek.java   | 33 ++++++++++++++++++++++
 .../fs/contract/s3a/ITestS3AContractSeek.java      | 32 +++++++++++++++++++++
 3 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index ccc86d0..35dd834 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -261,7 +261,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
       long forwardSeekLimit = Math.min(remainingInCurrentRequest,
           forwardSeekRange);
       boolean skipForward = remainingInCurrentRequest > 0
-          && diff <= forwardSeekLimit;
+          && diff < forwardSeekLimit;
       if (skipForward) {
         // the forward seek range is within the limits
         LOG.debug("Forward seek on {}, of {} bytes", uri, diff);
@@ -275,6 +275,8 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
 
         if (pos == targetPos) {
           // all is well
+          LOG.debug("Now at {}: bytes remaining in current request: {}",
+              pos, remainingInCurrentRequest());
           return;
         } else {
           // log a warning; continue to attempt to re-open
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRandomSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRandomSeek.java
new file mode 100644
index 0000000..e1ed976
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRandomSeek.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.Constants;
+
+public class ITestS3AContractRandomSeek extends ITestS3AContractSeek {
+
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    conf.setInt(Constants.READAHEAD_RANGE, READAHEAD);
+    conf.set(Constants.INPUT_FADVISE, Constants.INPUT_FADV_RANDOM);
+    return conf;
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
index 379ace8..d968e54 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
@@ -18,9 +18,15 @@
 
 package org.apache.hadoop.fs.contract.s3a;
 
+import org.junit.Test;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
 
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
 
@@ -29,6 +35,8 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
  */
 public class ITestS3AContractSeek extends AbstractContractSeekTest {
 
+  protected static final int READAHEAD = 1024;
+  
   /**
    * Create a configuration, possibly patching in S3Guard options.
    * @return a configuration
@@ -45,4 +53,28 @@ public class ITestS3AContractSeek extends AbstractContractSeekTest {
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
   }
+
+  /**
+   * Test for HADOOP-16109: Parquet reading S3AFileSystem causes EOF.
+   */
+  @Test
+  public void testReadPastReadahead() throws Throwable {
+    Path path = path("testReadPastReadahead");
+    byte[] dataset = ContractTestUtils.dataset(READAHEAD * 2, 'a', 32);
+    FileSystem fs = getFileSystem();
+    ContractTestUtils.writeDataset(fs, path, dataset, dataset.length, READAHEAD,
+        true);
+    // forward seek reading across readahead boundary
+    try (FSDataInputStream in = fs.open(path)) {
+      final byte[] temp = new byte[5];
+      in.readByte();
+      in.readFully(1023, temp); // <-- works
+    }
+    // forward seek reading from end of readahead boundary
+    try (FSDataInputStream in = fs.open(path)) {
+      final byte[] temp = new byte[5];
+      in.readByte();
+      in.readFully(1024, temp); // <-- throws EOFException
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org