You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2018/10/24 04:50:00 UTC
hadoop git commit: HADOOP-15822. zstd compressor can fail with a
small output buffer. Contributed by Jason Lowe.
Repository: hadoop
Updated Branches:
refs/heads/trunk a0c0b793c -> 8f97d6f2c
HADOOP-15822. zstd compressor can fail with a small output buffer. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f97d6f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f97d6f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f97d6f2
Branch: refs/heads/trunk
Commit: 8f97d6f2cdfccefba5457ae3d561e9ce0109da3f
Parents: a0c0b79
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 24 13:49:28 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 24 13:49:28 2018 +0900
----------------------------------------------------------------------
dev-support/docker/Dockerfile | 1 +
.../hadoop/io/compress/zstd/ZStandardDecompressor.java | 4 ++--
.../apache/hadoop/io/compress/zstd/ZStandardCompressor.c | 11 +++++++----
.../hadoop/io/compress/zstd/ZStandardDecompressor.c | 1 +
.../zstd/TestZStandardCompressorDecompressor.java | 10 ++++------
5 files changed, 15 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f97d6f2/dev-support/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 7679500..67140c9 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -63,6 +63,7 @@ RUN apt-get -q update \
libsnappy-dev \
libssl-dev \
libtool \
+ libzstd1-dev \
locales \
make \
pinentry-curses \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f97d6f2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
index 73d73e1..bc9d29c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
@@ -262,8 +262,8 @@ public class ZStandardDecompressor implements Decompressor {
int originalPosition = dst.position();
int n = inflateBytesDirect(
- src, src.position(), src.remaining(), dst, dst.position(),
- dst.remaining()
+ src, src.position(), src.limit(), dst, dst.position(),
+ dst.limit()
);
dst.position(originalPosition + n);
if (bytesInCompressedBuffer > 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f97d6f2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
index 055683a..289554b 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
@@ -195,10 +195,13 @@ JNIEXPORT jint Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_defla
ZSTD_inBuffer input = { uncompressed_bytes, uncompressed_direct_buf_len, uncompressed_direct_buf_off };
ZSTD_outBuffer output = { compressed_bytes, compressed_direct_buf_len, 0 };
- size_t size = dlsym_ZSTD_compressStream(stream, &output, &input);
- if (dlsym_ZSTD_isError(size)) {
- THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
- return (jint) 0;
+ size_t size;
+ if (uncompressed_direct_buf_len != 0) {
+ size = dlsym_ZSTD_compressStream(stream, &output, &input);
+ if (dlsym_ZSTD_isError(size)) {
+ THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
+ return (jint) 0;
+ }
}
if (finish && input.pos == input.size) {
// end the stream, flush and write the frame epilogue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f97d6f2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
index b97dd17..e28359b 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
@@ -178,6 +178,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompre
return (jint) 0;
}
uncompressed_bytes = ((char*) uncompressed_bytes) + uncompressed_direct_buf_off;
+ uncompressed_direct_buf_len -= uncompressed_direct_buf_off;
ZSTD_inBuffer input = { compressed_bytes, compressed_direct_buf_len, compressed_direct_buf_off };
ZSTD_outBuffer output = { uncompressed_bytes, uncompressed_direct_buf_len, 0 };
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f97d6f2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
index 04def24..dcfb7e9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
@@ -414,13 +414,11 @@ public class TestZStandardCompressorDecompressor {
outBuf.clear();
while (!decompressor.finished()) {
decompressor.decompress(inBuf, outBuf);
- if (outBuf.remaining() == 0) {
- outBuf.flip();
- while (outBuf.remaining() > 0) {
- assertEquals(expected.get(), outBuf.get());
- }
- outBuf.clear();
+ outBuf.flip();
+ while (outBuf.remaining() > 0) {
+ assertEquals(expected.get(), outBuf.get());
}
+ outBuf.clear();
}
outBuf.flip();
while (outBuf.remaining() > 0) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org