You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2018/11/21 19:06:51 UTC
[37/50] [abbrv] hadoop git commit: HDDS-732. Add read method which
takes offset and length in SignedChunkInputStream. Contributed by chencan.
HDDS-732. Add read method which takes offset and length in SignedChunkInputStream. Contributed by chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34b6fa7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34b6fa7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34b6fa7d
Branch: refs/heads/HDFS-13891
Commit: 34b6fa7d6b22439f5c703bf7e82de5c40c77994a
Parents: 22867de
Author: Márton Elek <el...@apache.org>
Authored: Wed Nov 21 17:59:20 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Wed Nov 21 17:59:36 2018 +0100
----------------------------------------------------------------------
.../ozone/s3/SignedChunksInputStream.java | 40 ++++++++++++++++++++
.../ozone/s3/TestSignedChunksInputStream.java | 30 +++++++++++++++
2 files changed, 70 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b6fa7d/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
index a35133f..b0dfce6 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
@@ -65,6 +65,46 @@ public class SignedChunksInputStream extends InputStream {
}
}
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ if (b == null) {
+ throw new NullPointerException();
+ } else if (off < 0 || len < 0 || len > b.length - off) {
+ throw new IndexOutOfBoundsException();
+ } else if (len == 0) {
+ return 0;
+ }
+ int currentOff = off;
+ int currentLen = len;
+ int totalReadBytes = 0;
+ int realReadLen = 0;
+ int maxReadLen = 0;
+ do {
+ if (remainingData > 0) {
+ maxReadLen = Math.min(remainingData, currentLen);
+ realReadLen = originalStream.read(b, currentOff, maxReadLen);
+ if (realReadLen == -1) {
+ break;
+ }
+ currentOff += realReadLen;
+ currentLen -= realReadLen;
+ totalReadBytes += realReadLen;
+ remainingData -= realReadLen;
+ if (remainingData == 0) {
+ //read the "\r\n" at the end of the data section
+ originalStream.read();
+ originalStream.read();
+ }
+ } else {
+ remainingData = readHeader();
+ if (remainingData == -1) {
+ break;
+ }
+ }
+ } while (currentLen > 0);
+ return totalReadBytes > 0 ? totalReadBytes : -1;
+ }
+
private int readHeader() throws IOException {
int prev = -1;
int curr = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b6fa7d/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
index f1af7cb..3599c05 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
@@ -49,32 +49,62 @@ public class TestSignedChunksInputStream {
@Test
public void singlechunk() throws IOException {
+ //test simple read()
InputStream is = fileContent("0A;chunk-signature"
+
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ "\n1234567890\r\n");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890", result);
+
+ //test read(byte[],int,int)
+ is = fileContent("0A;chunk-signature"
+ +
+ "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ + "\n1234567890\r\n");
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ Assert.assertEquals("1234567890", new String(bytes));
}
@Test
public void singlechunkwithoutend() throws IOException {
+ //test simple read()
InputStream is = fileContent("0A;chunk-signature"
+
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ "\n1234567890");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890", result);
+
+ //test read(byte[],int,int)
+ is = fileContent("0A;chunk-signature"
+ +
+ "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ + "\n1234567890");
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ Assert.assertEquals("1234567890", new String(bytes));
}
@Test
public void multichunks() throws IOException {
+ //test simple read()
InputStream is = fileContent("0a;chunk-signature=signature\r\n"
+ "1234567890\r\n"
+ "05;chunk-signature=signature\r\n"
+ "abcde\r\n");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890abcde", result);
+
+ //test read(byte[],int,int)
+ is = fileContent("0a;chunk-signature=signature\r\n"
+ + "1234567890\r\n"
+ + "05;chunk-signature=signature\r\n"
+ + "abcde\r\n");
+ byte[] bytes = new byte[15];
+ IOUtils.read(is, bytes, 0, 15);
+ Assert.assertEquals("1234567890abcde", new String(bytes));
}
private InputStream fileContent(String content) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org