You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/11/25 21:56:00 UTC
[arrow] branch master updated: ARROW-3831: [C++] Add support for
returning decompressed size
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 1013a1d ARROW-3831: [C++] Add support for returning decompressed size
1013a1d is described below
commit 1013a1dcd1904a4eab036a772184dd69e228f845
Author: Kouhei Sutou <ko...@clear-code.com>
AuthorDate: Sun Nov 25 16:55:52 2018 -0500
ARROW-3831: [C++] Add support for returning decompressed size
Author: Kouhei Sutou <ko...@clear-code.com>
Closes #3024 from kou/cpp-codec-decompressed-size and squashes the following commits:
9b16e585f <Kouhei Sutou> Update variable name
650ba3080 <Kouhei Sutou> Support output buffer is zero case with zlib
7f8f10ee4 <Kouhei Sutou> Fix variable
644d56518 <Kouhei Sutou> Add support for returning decompressed size
---
cpp/src/arrow/util/compression-test.cc | 16 ++++++++++++
cpp/src/arrow/util/compression.h | 25 ++++++++++++++++---
cpp/src/arrow/util/compression_brotli.cc | 21 +++++++++++-----
cpp/src/arrow/util/compression_brotli.h | 7 ++++--
cpp/src/arrow/util/compression_bz2.cc | 12 ++++++---
cpp/src/arrow/util/compression_bz2.h | 7 ++++--
cpp/src/arrow/util/compression_lz4.cc | 21 +++++++++++-----
cpp/src/arrow/util/compression_lz4.h | 7 ++++--
cpp/src/arrow/util/compression_snappy.cc | 26 ++++++++++++++-----
cpp/src/arrow/util/compression_snappy.h | 7 ++++--
cpp/src/arrow/util/compression_zlib.cc | 43 +++++++++++++++++++++-----------
cpp/src/arrow/util/compression_zlib.h | 7 ++++--
cpp/src/arrow/util/compression_zstd.cc | 25 +++++++++++++------
cpp/src/arrow/util/compression_zstd.h | 7 ++++--
14 files changed, 172 insertions(+), 59 deletions(-)
diff --git a/cpp/src/arrow/util/compression-test.cc b/cpp/src/arrow/util/compression-test.cc
index 899238b..f3e1106 100644
--- a/cpp/src/arrow/util/compression-test.cc
+++ b/cpp/src/arrow/util/compression-test.cc
@@ -79,6 +79,14 @@ void CheckCodecRoundtrip(Compression::type ctype, const vector<uint8_t>& data) {
ASSERT_EQ(data, decompressed);
+ // decompress with size with c2
+ int64_t actual_decompressed_size;
+ ASSERT_OK(c2->Decompress(compressed.size(), compressed.data(), decompressed.size(),
+ decompressed.data(), &actual_decompressed_size));
+
+ ASSERT_EQ(data, decompressed);
+ ASSERT_EQ(data.size(), actual_decompressed_size);
+
// compress with c2
int64_t actual_size2;
ASSERT_OK(c2->Compress(data.size(), data.data(), max_compressed_len, compressed.data(),
@@ -90,6 +98,14 @@ void CheckCodecRoundtrip(Compression::type ctype, const vector<uint8_t>& data) {
decompressed.data()));
ASSERT_EQ(data, decompressed);
+
+ // decompress with size with c1
+ int64_t actual_decompressed_size2;
+ ASSERT_OK(c1->Decompress(compressed.size(), compressed.data(), decompressed.size(),
+ decompressed.data(), &actual_decompressed_size2));
+
+ ASSERT_EQ(data, decompressed);
+ ASSERT_EQ(data.size(), actual_decompressed_size2);
}
// Check the streaming compressor against one-shot decompression
diff --git a/cpp/src/arrow/util/compression.h b/cpp/src/arrow/util/compression.h
index f935905..43174f4 100644
--- a/cpp/src/arrow/util/compression.h
+++ b/cpp/src/arrow/util/compression.h
@@ -98,13 +98,30 @@ class ARROW_EXPORT Codec {
/// \brief One-shot decompression function
///
- /// output_len must be correct and therefore be obtained in advance.
+ /// output_buffer_len must be correct and therefore be obtained in advance.
///
/// \note One-shot decompression is not always compatible with streaming
/// compression. Depending on the codec (e.g. LZ4), different formats may
/// be used.
- virtual Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
- uint8_t* output_buffer) = 0;
+ virtual Status Decompress(int64_t input_len, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output_buffer) = 0;
+
+ /// \brief One-shot decompression function that also returns the
+ /// actual decompressed size.
+ ///
+ /// \param[in] input_len the number of bytes of compressed data.
+ /// \param[in] input the compressed data.
+ /// \param[in] output_buffer_len the number of bytes of buffer for
+ /// decompressed data.
+ /// \param[in] output_buffer the buffer for decompressed data.
+ /// \param[out] output_len the actual decompressed size.
+ ///
+ /// \note One-shot decompression is not always compatible with streaming
+ /// compression. Depending on the codec (e.g. LZ4), different formats may
+ /// be used.
+ virtual Status Decompress(int64_t input_len, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output_buffer,
+ int64_t* output_len) = 0;
/// \brief One-shot compression function
///
@@ -115,7 +132,7 @@ class ARROW_EXPORT Codec {
/// be used.
virtual Status Compress(int64_t input_len, const uint8_t* input,
int64_t output_buffer_len, uint8_t* output_buffer,
- int64_t* output_length) = 0;
+ int64_t* output_len) = 0;
virtual int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) = 0;
diff --git a/cpp/src/arrow/util/compression_brotli.cc b/cpp/src/arrow/util/compression_brotli.cc
index 1b8ab85..89d099d 100644
--- a/cpp/src/arrow/util/compression_brotli.cc
+++ b/cpp/src/arrow/util/compression_brotli.cc
@@ -199,12 +199,21 @@ Status BrotliCodec::MakeDecompressor(std::shared_ptr<Decompressor>* out) {
}
Status BrotliCodec::Decompress(int64_t input_len, const uint8_t* input,
- int64_t output_len, uint8_t* output_buffer) {
- std::size_t output_size = output_len;
+ int64_t output_buffer_len, uint8_t* output_buffer) {
+ return Decompress(input_len, input, output_buffer_len, output_buffer, nullptr);
+}
+
+Status BrotliCodec::Decompress(int64_t input_len, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output_buffer,
+ int64_t* output_len) {
+ std::size_t output_size = output_buffer_len;
if (BrotliDecoderDecompress(input_len, input, &output_size, output_buffer) !=
BROTLI_DECODER_RESULT_SUCCESS) {
return Status::IOError("Corrupt brotli compressed data.");
}
+ if (output_len) {
+ *output_len = output_size;
+ }
return Status::OK();
}
@@ -215,14 +224,14 @@ int64_t BrotliCodec::MaxCompressedLen(int64_t input_len,
Status BrotliCodec::Compress(int64_t input_len, const uint8_t* input,
int64_t output_buffer_len, uint8_t* output_buffer,
- int64_t* output_length) {
- std::size_t output_len = output_buffer_len;
+ int64_t* output_len) {
+ std::size_t output_size = output_buffer_len;
if (BrotliEncoderCompress(kBrotliDefaultCompressionLevel, BROTLI_DEFAULT_WINDOW,
- BROTLI_DEFAULT_MODE, input_len, input, &output_len,
+ BROTLI_DEFAULT_MODE, input_len, input, &output_size,
output_buffer) == BROTLI_FALSE) {
return Status::IOError("Brotli compression failure.");
}
- *output_length = output_len;
+ *output_len = output_size;
return Status::OK();
}
diff --git a/cpp/src/arrow/util/compression_brotli.h b/cpp/src/arrow/util/compression_brotli.h
index 0b403ee..59f97cd 100644
--- a/cpp/src/arrow/util/compression_brotli.h
+++ b/cpp/src/arrow/util/compression_brotli.h
@@ -31,11 +31,14 @@ namespace util {
// Brotli codec.
class ARROW_EXPORT BrotliCodec : public Codec {
public:
- Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
uint8_t* output_buffer) override;
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+ uint8_t* output_buffer, int64_t* output_len) override;
+
Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
- uint8_t* output_buffer, int64_t* output_length) override;
+ uint8_t* output_buffer, int64_t* output_len) override;
int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
diff --git a/cpp/src/arrow/util/compression_bz2.cc b/cpp/src/arrow/util/compression_bz2.cc
index ae78f98..407fb82 100644
--- a/cpp/src/arrow/util/compression_bz2.cc
+++ b/cpp/src/arrow/util/compression_bz2.cc
@@ -238,8 +238,14 @@ Status BZ2Codec::MakeDecompressor(std::shared_ptr<Decompressor>* out) {
return Status::OK();
}
-Status BZ2Codec::Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
- uint8_t* output_buffer) {
+Status BZ2Codec::Decompress(int64_t input_len, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output_buffer) {
+ return Status::NotImplemented("One-shot bz2 decompression not supported");
+}
+
+Status BZ2Codec::Decompress(int64_t input_len, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output_buffer,
+ int64_t* output_len) {
return Status::NotImplemented("One-shot bz2 decompression not supported");
}
@@ -251,7 +257,7 @@ int64_t BZ2Codec::MaxCompressedLen(int64_t input_len,
Status BZ2Codec::Compress(int64_t input_len, const uint8_t* input,
int64_t output_buffer_len, uint8_t* output_buffer,
- int64_t* output_length) {
+ int64_t* output_len) {
return Status::NotImplemented("One-shot bz2 compression not supported");
}
diff --git a/cpp/src/arrow/util/compression_bz2.h b/cpp/src/arrow/util/compression_bz2.h
index 304f4c0..2146158 100644
--- a/cpp/src/arrow/util/compression_bz2.h
+++ b/cpp/src/arrow/util/compression_bz2.h
@@ -31,11 +31,14 @@ namespace util {
// BZ2 codec.
class ARROW_EXPORT BZ2Codec : public Codec {
public:
- Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
uint8_t* output_buffer) override;
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+ uint8_t* output_buffer, int64_t* output_len) override;
+
Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
- uint8_t* output_buffer, int64_t* output_length) override;
+ uint8_t* output_buffer, int64_t* output_len) override;
int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
diff --git a/cpp/src/arrow/util/compression_lz4.cc b/cpp/src/arrow/util/compression_lz4.cc
index 7ba1f12..0acd54d 100644
--- a/cpp/src/arrow/util/compression_lz4.cc
+++ b/cpp/src/arrow/util/compression_lz4.cc
@@ -250,14 +250,23 @@ Status Lz4Codec::MakeDecompressor(std::shared_ptr<Decompressor>* out) {
return Status::OK();
}
-Status Lz4Codec::Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
- uint8_t* output_buffer) {
+Status Lz4Codec::Decompress(int64_t input_len, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output_buffer) {
+ return Decompress(input_len, input, output_buffer_len, output_buffer, nullptr);
+}
+
+Status Lz4Codec::Decompress(int64_t input_len, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output_buffer,
+ int64_t* output_len) {
int64_t decompressed_size = LZ4_decompress_safe(
reinterpret_cast<const char*>(input), reinterpret_cast<char*>(output_buffer),
- static_cast<int>(input_len), static_cast<int>(output_len));
+ static_cast<int>(input_len), static_cast<int>(output_buffer_len));
if (decompressed_size < 0) {
return Status::IOError("Corrupt Lz4 compressed data.");
}
+ if (output_len) {
+ *output_len = decompressed_size;
+ }
return Status::OK();
}
@@ -268,11 +277,11 @@ int64_t Lz4Codec::MaxCompressedLen(int64_t input_len,
Status Lz4Codec::Compress(int64_t input_len, const uint8_t* input,
int64_t output_buffer_len, uint8_t* output_buffer,
- int64_t* output_length) {
- *output_length = LZ4_compress_default(
+ int64_t* output_len) {
+ *output_len = LZ4_compress_default(
reinterpret_cast<const char*>(input), reinterpret_cast<char*>(output_buffer),
static_cast<int>(input_len), static_cast<int>(output_buffer_len));
- if (*output_length == 0) {
+ if (*output_len == 0) {
return Status::IOError("Lz4 compression failure.");
}
return Status::OK();
diff --git a/cpp/src/arrow/util/compression_lz4.h b/cpp/src/arrow/util/compression_lz4.h
index 8c4bcf5..4d06f03 100644
--- a/cpp/src/arrow/util/compression_lz4.h
+++ b/cpp/src/arrow/util/compression_lz4.h
@@ -31,11 +31,14 @@ namespace util {
// Lz4 codec.
class ARROW_EXPORT Lz4Codec : public Codec {
public:
- Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
uint8_t* output_buffer) override;
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+ uint8_t* output_buffer, int64_t* output_len) override;
+
Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
- uint8_t* output_buffer, int64_t* output_length) override;
+ uint8_t* output_buffer, int64_t* output_len) override;
int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
diff --git a/cpp/src/arrow/util/compression_snappy.cc b/cpp/src/arrow/util/compression_snappy.cc
index cc55408..ab93c3c 100644
--- a/cpp/src/arrow/util/compression_snappy.cc
+++ b/cpp/src/arrow/util/compression_snappy.cc
@@ -42,8 +42,22 @@ Status SnappyCodec::MakeDecompressor(std::shared_ptr<Decompressor>* out) {
}
Status SnappyCodec::Decompress(int64_t input_len, const uint8_t* input,
- int64_t ARROW_ARG_UNUSED(output_len),
- uint8_t* output_buffer) {
+ int64_t output_buffer_len, uint8_t* output_buffer) {
+ return Decompress(input_len, input, output_buffer_len, output_buffer, nullptr);
+}
+
+Status SnappyCodec::Decompress(int64_t input_len, const uint8_t* input,
+ int64_t ARROW_ARG_UNUSED(output_buffer_len),
+ uint8_t* output_buffer, int64_t* output_len) {
+ if (output_len) {
+ size_t decompressed_size;
+ if (!snappy::GetUncompressedLength(reinterpret_cast<const char*>(input),
+ static_cast<size_t>(input_len),
+ &decompressed_size)) {
+ return Status::IOError("Corrupt snappy compressed data.");
+ }
+ *output_len = static_cast<int64_t>(decompressed_size);
+ }
if (!snappy::RawUncompress(reinterpret_cast<const char*>(input),
static_cast<size_t>(input_len),
reinterpret_cast<char*>(output_buffer))) {
@@ -59,12 +73,12 @@ int64_t SnappyCodec::MaxCompressedLen(int64_t input_len,
Status SnappyCodec::Compress(int64_t input_len, const uint8_t* input,
int64_t ARROW_ARG_UNUSED(output_buffer_len),
- uint8_t* output_buffer, int64_t* output_length) {
- size_t output_len;
+ uint8_t* output_buffer, int64_t* output_len) {
+ size_t output_size;
snappy::RawCompress(reinterpret_cast<const char*>(input),
static_cast<size_t>(input_len),
- reinterpret_cast<char*>(output_buffer), &output_len);
- *output_length = static_cast<int64_t>(output_len);
+ reinterpret_cast<char*>(output_buffer), &output_size);
+ *output_len = static_cast<int64_t>(output_size);
return Status::OK();
}
diff --git a/cpp/src/arrow/util/compression_snappy.h b/cpp/src/arrow/util/compression_snappy.h
index 722f355..7029400 100644
--- a/cpp/src/arrow/util/compression_snappy.h
+++ b/cpp/src/arrow/util/compression_snappy.h
@@ -30,11 +30,14 @@ namespace util {
class ARROW_EXPORT SnappyCodec : public Codec {
public:
- Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
uint8_t* output_buffer) override;
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+ uint8_t* output_buffer, int64_t* output_len) override;
+
Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
- uint8_t* output_buffer, int64_t* output_length) override;
+ uint8_t* output_buffer, int64_t* output_len) override;
int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
diff --git a/cpp/src/arrow/util/compression_zlib.cc b/cpp/src/arrow/util/compression_zlib.cc
index 9fe163e..686dffa 100644
--- a/cpp/src/arrow/util/compression_zlib.cc
+++ b/cpp/src/arrow/util/compression_zlib.cc
@@ -382,16 +382,20 @@ class GZipCodec::GZipCodecImpl {
decompressor_initialized_ = false;
}
- Status Decompress(int64_t input_length, const uint8_t* input, int64_t output_length,
- uint8_t* output) {
+ Status Decompress(int64_t input_length, const uint8_t* input,
+ int64_t output_buffer_length, uint8_t* output,
+ int64_t* output_length) {
if (!decompressor_initialized_) {
RETURN_NOT_OK(InitDecompressor());
}
- if (output_length == 0) {
- // The zlib library does not allow *output to be NULL, even when output_length
- // is 0 (inflate() will return Z_STREAM_ERROR). We don't consider this an
- // error, so bail early if no output is expected. Note that we don't signal
- // an error if the input actually contains compressed data.
+ if (output_buffer_length == 0) {
+ // The zlib library does not allow *output to be NULL, even when
+ // output_buffer_length is 0 (inflate() will return Z_STREAM_ERROR). We don't
+ // consider this an error, so bail early if no output is expected. Note that we
+ // don't signal an error if the input actually contains compressed data.
+ if (output_length) {
+ *output_length = 0;
+ }
return Status::OK();
}
@@ -413,7 +417,7 @@ class GZipCodec::GZipCodecImpl {
stream_.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(input));
stream_.avail_in = static_cast<uInt>(input_length);
stream_.next_out = reinterpret_cast<Bytef*>(output);
- stream_.avail_out = static_cast<uInt>(output_length);
+ stream_.avail_out = static_cast<uInt>(output_buffer_length);
// We know the output size. In this case, we can use Z_FINISH
// which is more efficient.
@@ -423,7 +427,7 @@ class GZipCodec::GZipCodecImpl {
// Failure, buffer was too small
std::stringstream ss;
ss << "Too small a buffer passed to GZipCodec. InputLength=" << input_length
- << " OutputLength=" << output_length;
+ << " OutputLength=" << output_buffer_length;
return Status::IOError(ss.str());
}
@@ -434,6 +438,11 @@ class GZipCodec::GZipCodecImpl {
if (stream_.msg != NULL) ss << stream_.msg;
return Status::IOError(ss.str());
}
+
+ if (output_length) {
+ *output_length = stream_.total_out;
+ }
+
return Status::OK();
}
@@ -450,7 +459,7 @@ class GZipCodec::GZipCodecImpl {
}
Status Compress(int64_t input_length, const uint8_t* input, int64_t output_buffer_len,
- uint8_t* output, int64_t* output_length) {
+ uint8_t* output, int64_t* output_len) {
if (!compressor_initialized_) {
RETURN_NOT_OK(InitCompressor());
}
@@ -478,7 +487,7 @@ class GZipCodec::GZipCodecImpl {
}
// Actual output length
- *output_length = output_buffer_len - stream_.avail_out;
+ *output_len = output_buffer_len - stream_.avail_out;
return Status::OK();
}
@@ -508,7 +517,13 @@ GZipCodec::~GZipCodec() {}
Status GZipCodec::Decompress(int64_t input_length, const uint8_t* input,
int64_t output_buffer_len, uint8_t* output) {
- return impl_->Decompress(input_length, input, output_buffer_len, output);
+ return impl_->Decompress(input_length, input, output_buffer_len, output, nullptr);
+}
+
+Status GZipCodec::Decompress(int64_t input_length, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output,
+ int64_t* output_len) {
+ return impl_->Decompress(input_length, input, output_buffer_len, output, output_len);
}
int64_t GZipCodec::MaxCompressedLen(int64_t input_length, const uint8_t* input) {
@@ -517,8 +532,8 @@ int64_t GZipCodec::MaxCompressedLen(int64_t input_length, const uint8_t* input)
Status GZipCodec::Compress(int64_t input_length, const uint8_t* input,
int64_t output_buffer_len, uint8_t* output,
- int64_t* output_length) {
- return impl_->Compress(input_length, input, output_buffer_len, output, output_length);
+ int64_t* output_len) {
+ return impl_->Compress(input_length, input, output_buffer_len, output, output_len);
}
Status GZipCodec::MakeCompressor(std::shared_ptr<Compressor>* out) {
diff --git a/cpp/src/arrow/util/compression_zlib.h b/cpp/src/arrow/util/compression_zlib.h
index f934198..9a5feaa 100644
--- a/cpp/src/arrow/util/compression_zlib.h
+++ b/cpp/src/arrow/util/compression_zlib.h
@@ -41,11 +41,14 @@ class ARROW_EXPORT GZipCodec : public Codec {
explicit GZipCodec(Format format = GZIP);
~GZipCodec() override;
- Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
uint8_t* output_buffer) override;
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+ uint8_t* output_buffer, int64_t* output_len) override;
+
Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
- uint8_t* output_buffer, int64_t* output_length) override;
+ uint8_t* output_buffer, int64_t* output_len) override;
int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;
diff --git a/cpp/src/arrow/util/compression_zstd.cc b/cpp/src/arrow/util/compression_zstd.cc
index 8bf02be..083cae9 100644
--- a/cpp/src/arrow/util/compression_zstd.cc
+++ b/cpp/src/arrow/util/compression_zstd.cc
@@ -199,24 +199,33 @@ Status ZSTDCodec::MakeDecompressor(std::shared_ptr<Decompressor>* out) {
return Status::OK();
}
-Status ZSTDCodec::Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
- uint8_t* output_buffer) {
+Status ZSTDCodec::Decompress(int64_t input_len, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output_buffer) {
+ return Decompress(input_len, input, output_buffer_len, output_buffer, nullptr);
+}
+
+Status ZSTDCodec::Decompress(int64_t input_len, const uint8_t* input,
+ int64_t output_buffer_len, uint8_t* output_buffer,
+ int64_t* output_len) {
if (output_buffer == nullptr) {
// We may pass a NULL 0-byte output buffer but some zstd versions demand
// a valid pointer: https://github.com/facebook/zstd/issues/1385
static uint8_t empty_buffer[1];
- DCHECK_EQ(output_len, 0);
+ DCHECK_EQ(output_buffer_len, 0);
output_buffer = empty_buffer;
}
- size_t ret = ZSTD_decompress(output_buffer, static_cast<size_t>(output_len), input,
- static_cast<size_t>(input_len));
+ size_t ret = ZSTD_decompress(output_buffer, static_cast<size_t>(output_buffer_len),
+ input, static_cast<size_t>(input_len));
if (ZSTD_isError(ret)) {
return ZSTDError(ret, "ZSTD decompression failed: ");
}
- if (static_cast<int64_t>(ret) != output_len) {
+ if (static_cast<int64_t>(ret) != output_buffer_len) {
return Status::IOError("Corrupt ZSTD compressed data.");
}
+ if (output_len) {
+ *output_len = static_cast<int64_t>(ret);
+ }
return Status::OK();
}
@@ -227,14 +236,14 @@ int64_t ZSTDCodec::MaxCompressedLen(int64_t input_len,
Status ZSTDCodec::Compress(int64_t input_len, const uint8_t* input,
int64_t output_buffer_len, uint8_t* output_buffer,
- int64_t* output_length) {
+ int64_t* output_len) {
size_t ret =
ZSTD_compress(output_buffer, static_cast<size_t>(output_buffer_len), input,
static_cast<size_t>(input_len), kZSTDDefaultCompressionLevel);
if (ZSTD_isError(ret)) {
return ZSTDError(ret, "ZSTD compression failed: ");
}
- *output_length = static_cast<int64_t>(ret);
+ *output_len = static_cast<int64_t>(ret);
return Status::OK();
}
diff --git a/cpp/src/arrow/util/compression_zstd.h b/cpp/src/arrow/util/compression_zstd.h
index 06da152..8b05d8c 100644
--- a/cpp/src/arrow/util/compression_zstd.h
+++ b/cpp/src/arrow/util/compression_zstd.h
@@ -31,11 +31,14 @@ namespace util {
// ZSTD codec.
class ARROW_EXPORT ZSTDCodec : public Codec {
public:
- Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
uint8_t* output_buffer) override;
+ Status Decompress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
+ uint8_t* output_buffer, int64_t* output_len) override;
+
Status Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
- uint8_t* output_buffer, int64_t* output_length) override;
+ uint8_t* output_buffer, int64_t* output_len) override;
int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) override;