[GitHub] [ozone] umamaheswararao commented on pull request #3094: HDDS-6319. EC: Fix read big file failure with EC policy 10+4.

[GitBox <gi...@apache.org>]

umamaheswararao commented on pull request #3094:
URL: https://github.com/apache/ozone/pull/3094#issuecomment-1041183210


   Thanks @guihecheng for the patch. Is there a way we can add some test around this? 
   Below test can reproduce this case (Probably you may want to keep this test in appropriate  test class and If there is a better way, that would be awesome though, thanks CC: @sodonnel ):
   ```
   diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
   index 46a7b5099..f6e7a6dab 100644
   --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
   +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java
   @@ -210,6 +210,11 @@ public void addStream(BlockInputStream blockInputStream) {
        blockStreams.add(blockInputStream);
      }
   
   +  @VisibleForTesting
   +  public void addStream(BlockExtendedInputStream blockInputStream) {
   +    blockStreams.add(blockInputStream);
   +  }
   +
      /**
       * {@inheritDoc}
       */
   @@ -264,6 +269,7 @@ protected synchronized int readWithStrategy(ByteReaderStrategy strategy)
   
          // Get the current blockStream and read data from it
          BlockExtendedInputStream current = blockStreams.get(blockIndex);
   +
          int numBytesToRead = Math.min(buffLen, (int)current.getRemaining());
          int numBytesRead = strategy.readFromBlock(current, numBytesToRead);
          if (numBytesRead != numBytesToRead) {
   diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockInputStream.java
   index b3b01ba4a..4289728a0 100644
   --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockInputStream.java
   +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestECBlockInputStream.java
   @@ -30,6 +30,7 @@
    import org.apache.hadoop.ozone.client.io.BadDataLocationException;
    import org.apache.hadoop.ozone.client.io.BlockInputStreamFactory;
    import org.apache.hadoop.ozone.client.io.ECBlockInputStream;
   +import org.apache.hadoop.ozone.client.io.KeyInputStream;
    import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
    import org.apache.hadoop.security.token.Token;
    import org.junit.Assert;
   @@ -230,6 +231,25 @@ public void testSimpleRead() throws IOException {
        }
      }
   
   +  @Test
   +  public void testLargeSizeReadWithKeyInputStream() throws IOException {
   +    ECReplicationConfig ec10And4RepConfig =
   +        new ECReplicationConfig(10, 4, ECReplicationConfig.EcCodec.RS, ONEMB);
   +    OmKeyLocationInfo keyInfo = ECStreamTestUtil
   +        .createKeyInfo(ec10And4RepConfig, 14, (10 * 256L * ONEMB));
   +    try (ECBlockInputStream ecb = new ECBlockInputStream(ec10And4RepConfig,
   +        keyInfo, true, null, null, streamFactory)) {
   +
   +      KeyInputStream kis = new KeyInputStream();
   +      kis.addStream(ecb);
   +      ByteBuffer buf = ByteBuffer.allocate(100);
   +      int read = kis.read(buf);
   +      Assert.assertEquals(100, read);
   +      validateBufferContents(buf, 0, 100, (byte) 0);
   +      Assert.assertEquals(100, ecb.getPos());
   +    }
   +  }
   +
      /**
       * This test is to ensure we can read a small key of 1 chunk or less when only
       * the first replica index is available.
   ```


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org