You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by bl...@apache.org on 2014/09/23 21:14:23 UTC
git commit: PARQUET-82: Check page size is valid when writing.
Repository: incubator-parquet-mr
Updated Branches:
refs/heads/master 0eb963795 -> 59c58d0b8
PARQUET-82: Check page size is valid when writing.
Author: Ryan Blue <rb...@cloudera.com>
Closes #48 from rdblue/PARQUET-82-check-page-size and squashes the following commits:
9f31402 [Ryan Blue] PARQUET-82: Check page size is valid when writing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/59c58d0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/59c58d0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/59c58d0b
Branch: refs/heads/master
Commit: 59c58d0b829aa156f038cc900b803508f8849765
Parents: 0eb9637
Author: Ryan Blue <rb...@cloudera.com>
Authored: Tue Sep 23 12:14:17 2014 -0700
Committer: Ryan Blue <rb...@cloudera.com>
Committed: Tue Sep 23 12:14:17 2014 -0700
----------------------------------------------------------------------
.../hadoop/ColumnChunkPageWriteStore.java | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/59c58d0b/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java b/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java
index 279eb56..6d7f685 100644
--- a/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java
+++ b/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java
@@ -77,8 +77,18 @@ class ColumnChunkPageWriteStore implements PageWriteStore {
Encoding dlEncoding,
Encoding valuesEncoding) throws IOException {
long uncompressedSize = bytes.size();
+ if (uncompressedSize > Integer.MAX_VALUE) {
+ throw new ParquetEncodingException(
+ "Cannot write page larger than Integer.MAX_VALUE bytes: " +
+ uncompressedSize);
+ }
BytesInput compressedBytes = compressor.compress(bytes);
long compressedSize = compressedBytes.size();
+ if (compressedSize > Integer.MAX_VALUE) {
+ throw new ParquetEncodingException(
+ "Cannot write compressed page larger than Integer.MAX_VALUE bytes: "
+ + compressedSize);
+ }
BooleanStatistics statistics = new BooleanStatistics(); // dummy stats object
parquetMetadataConverter.writeDataPageHeader(
(int)uncompressedSize,
@@ -107,8 +117,18 @@ class ColumnChunkPageWriteStore implements PageWriteStore {
Encoding dlEncoding,
Encoding valuesEncoding) throws IOException {
long uncompressedSize = bytes.size();
+ if (uncompressedSize > Integer.MAX_VALUE) {
+ throw new ParquetEncodingException(
+ "Cannot write page larger than Integer.MAX_VALUE bytes: " +
+ uncompressedSize);
+ }
BytesInput compressedBytes = compressor.compress(bytes);
long compressedSize = compressedBytes.size();
+ if (compressedSize > Integer.MAX_VALUE) {
+ throw new ParquetEncodingException(
+ "Cannot write compressed page larger than Integer.MAX_VALUE bytes: "
+ + compressedSize);
+ }
parquetMetadataConverter.writeDataPageHeader(
(int)uncompressedSize,
(int)compressedSize,