You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2015/11/26 20:52:14 UTC
[04/50] [abbrv] hadoop git commit: HDFS-7988. Replace usage of
ExactSizeInputStream with LimitInputStream. Contributed by Walter Su.
HDFS-7988. Replace usage of ExactSizeInputStream with LimitInputStream. Contributed by Walter Su.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/298a8cb0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/298a8cb0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/298a8cb0
Branch: refs/heads/yarn-2877
Commit: 298a8cb096906b5d688842f6520e90dc9779f0b3
Parents: 70c2670
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Nov 23 10:50:12 2015 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Mon Nov 23 10:50:12 2015 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 4 +-
.../hadoop/hdfs/util/ExactSizeInputStream.java | 127 ------------------
.../hdfs/util/TestExactSizeInputStream.java | 129 -------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
4 files changed, 5 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/298a8cb0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 7ad9013..33bdc5c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -162,13 +162,13 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
-import org.apache.hadoop.hdfs.util.ExactSizeInputStream;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DataChecksum;
+import org.apache.hadoop.util.LimitInputStream;
/**
* Utilities for converting protobuf classes to and from hdfs-client side
@@ -401,7 +401,7 @@ public class PBHelperClient {
int size = CodedInputStream.readRawVarint32(firstByte, input);
assert size >= 0;
- return new ExactSizeInputStream(input, size);
+ return new LimitInputStream(input, size);
}
public static CipherOption convert(HdfsProtos.CipherOptionProto proto) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/298a8cb0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ExactSizeInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ExactSizeInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ExactSizeInputStream.java
deleted file mode 100644
index a0c59ac..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ExactSizeInputStream.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.util;
-
-import java.io.EOFException;
-import java.io.FilterInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import com.google.common.base.Preconditions;
-
-import javax.annotation.Nonnull;
-
-/**
- * An InputStream implementations which reads from some other InputStream
- * but expects an exact number of bytes. Any attempts to read past the
- * specified number of bytes will return as if the end of the stream
- * was reached. If the end of the underlying stream is reached prior to
- * the specified number of bytes, an EOFException is thrown.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class ExactSizeInputStream extends FilterInputStream {
- private int remaining;
-
- /**
- * Construct an input stream that will read no more than
- * 'numBytes' bytes.
- *
- * If an EOF occurs on the underlying stream before numBytes
- * bytes have been read, an EOFException will be thrown.
- *
- * @param in the inputstream to wrap
- * @param numBytes the number of bytes to read
- */
- public ExactSizeInputStream(InputStream in, int numBytes) {
- super(in);
- Preconditions.checkArgument(numBytes >= 0,
- "Negative expected bytes: ", numBytes);
- this.remaining = numBytes;
- }
-
- @Override
- public int available() throws IOException {
- return Math.min(super.available(), remaining);
- }
-
- @Override
- public int read() throws IOException {
- // EOF if we reached our limit
- if (remaining <= 0) {
- return -1;
- }
- final int result = super.read();
- if (result >= 0) {
- --remaining;
- } else if (remaining > 0) {
- // Underlying stream reached EOF but we haven't read the expected
- // number of bytes.
- throw new EOFException(
- "Premature EOF. Expected " + remaining + "more bytes");
- }
- return result;
- }
-
- @Override
- public int read(@Nonnull final byte[] b, final int off, int len)
- throws IOException {
- if (remaining <= 0) {
- return -1;
- }
- len = Math.min(len, remaining);
- final int result = super.read(b, off, len);
- if (result >= 0) {
- remaining -= result;
- } else if (remaining > 0) {
- // Underlying stream reached EOF but we haven't read the expected
- // number of bytes.
- throw new EOFException(
- "Premature EOF. Expected " + remaining + "more bytes");
- }
- return result;
- }
-
- @Override
- public long skip(final long n) throws IOException {
- final long result = super.skip(Math.min(n, remaining));
- if (result > 0) {
- remaining -= result;
- } else if (remaining > 0) {
- // Underlying stream reached EOF but we haven't read the expected
- // number of bytes.
- throw new EOFException(
- "Premature EOF. Expected " + remaining + "more bytes");
- }
- return result;
- }
-
- @Override
- public boolean markSupported() {
- return false;
- }
-
- @Override
- public void mark(int readlimit) {
- throw new UnsupportedOperationException();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/298a8cb0/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/util/TestExactSizeInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/util/TestExactSizeInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/util/TestExactSizeInputStream.java
deleted file mode 100644
index 02787be..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/util/TestExactSizeInputStream.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.junit.Test;
-
-public class TestExactSizeInputStream {
- @Test
- public void testBasicsReadSingle() throws IOException {
- ExactSizeInputStream s = new ExactSizeInputStream(byteStream("hello"), 3);
- assertEquals(3, s.available());
-
- assertEquals((int)'h', s.read());
- assertEquals((int)'e', s.read());
- assertEquals((int)'l', s.read());
- assertEquals(-1, s.read());
- assertEquals(0, s.available());
- }
-
- @Test
- public void testBasicsReadArray() throws IOException {
- ExactSizeInputStream s = new ExactSizeInputStream(byteStream("hello"), 3);
- assertEquals(3, s.available());
-
- byte[] buf = new byte[10];
-
- assertEquals(2, s.read(buf, 0, 2));
- assertEquals('h', buf[0]);
- assertEquals('e', buf[1]);
-
- assertEquals(1, s.read(buf, 0, 2));
- assertEquals('l', buf[0]);
-
- assertEquals(-1, s.read(buf, 0, 2));
- }
-
- @Test
- public void testBasicsSkip() throws IOException {
- ExactSizeInputStream s = new ExactSizeInputStream(byteStream("hello"), 3);
- assertEquals(3, s.available());
-
- assertEquals(2, s.skip(2));
- assertEquals(1, s.skip(2));
- assertEquals(0, s.skip(2));
- }
-
- @Test
- public void testReadNotEnough() throws IOException {
- // Ask for 5 bytes, only has 2
- ExactSizeInputStream s = new ExactSizeInputStream(byteStream("he"), 5);
- assertEquals(2, s.available());
-
- assertEquals((int)'h', s.read());
- assertEquals((int)'e', s.read());
- try {
- s.read();
- fail("Read when should be out of data");
- } catch (EOFException e) {
- // expected
- }
- }
-
- @Test
- public void testSkipNotEnough() throws IOException {
- // Ask for 5 bytes, only has 2
- ExactSizeInputStream s = new ExactSizeInputStream(byteStream("he"), 5);
- assertEquals(2, s.skip(3));
- try {
- s.skip(1);
- fail("Skip when should be out of data");
- } catch (EOFException e) {
- // expected
- }
- }
-
- @Test
- public void testReadArrayNotEnough() throws IOException {
- // Ask for 5 bytes, only has 2
- ExactSizeInputStream s = new ExactSizeInputStream(byteStream("he"), 5);
- byte[] buf = new byte[10];
- assertEquals(2, s.read(buf, 0, 5));
- try {
- s.read(buf, 2, 3);
- fail("Read buf when should be out of data");
- } catch (EOFException e) {
- // expected
- }
- }
-
- @Test
- public void testMark() throws IOException {
- ExactSizeInputStream s = new ExactSizeInputStream(byteStream("he"), 5);
- assertFalse(s.markSupported());
- try {
- s.mark(1);
- fail("Mark should not succeed");
- } catch (UnsupportedOperationException uoe) {
- // expected
- }
- }
-
- private static InputStream byteStream(String data) {
- return new ByteArrayInputStream(data.getBytes());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/298a8cb0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9016154..47b9b08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1670,6 +1670,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9153. Pretty-format the output for DFSIO. (Kai Zheng via wheat9)
+ HDFS-7988. Replace usage of ExactSizeInputStream with LimitInputStream.
+ (Walter Su via wheat9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than