You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ju...@apache.org on 2016/09/08 21:22:37 UTC
parquet-mr git commit: PARQUET-623: Fix DeltaByteArrayReader#skip.
Repository: parquet-mr
Updated Branches:
refs/heads/master 6dad1e3bd -> 044de16c1
PARQUET-623: Fix DeltaByteArrayReader#skip.
Previously, this passed the skip to the underlying readers, but would
not update previous and would corrupt values or cause exceptions.
Author: Ryan Blue <bl...@apache.org>
Closes #366 from rdblue/PARQUET-623-fix-delta-byte-array-skip and squashes the following commits:
f85800c [Ryan Blue] PARQUET-623: Fix DeltaByteArrayReader#skip.
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/044de16c
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/044de16c
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/044de16c
Branch: refs/heads/master
Commit: 044de16c14076019f87763b7b58c45664ee57c11
Parents: 6dad1e3
Author: Ryan Blue <bl...@apache.org>
Authored: Thu Sep 8 14:22:30 2016 -0700
Committer: Julien Le Dem <ju...@dremio.com>
Committed: Thu Sep 8 14:22:30 2016 -0700
----------------------------------------------------------------------
.../values/deltastrings/DeltaByteArrayReader.java | 4 ++--
.../values/deltastrings/TestDeltaByteArray.java | 18 ++++++++++++++++++
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/044de16c/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
index c2cfd6d..742b515 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
@@ -55,8 +55,8 @@ public class DeltaByteArrayReader extends ValuesReader implements RequiresPrevio
@Override
public void skip() {
- prefixLengthReader.skip();
- suffixReader.skip();
+ // read the next value to skip so that previous is correct.
+ readBytes();
}
@Override
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/044de16c/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
index 770f4dc..4f8f40c 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
@@ -19,6 +19,7 @@
package org.apache.parquet.column.values.deltastrings;
import java.io.IOException;
+import java.nio.ByteBuffer;
import org.junit.Test;
import org.junit.Assert;
@@ -50,6 +51,13 @@ public class TestDeltaByteArray {
}
@Test
+ public void testRandomStringsWithSkip() throws Exception {
+ DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
+ DeltaByteArrayReader reader = new DeltaByteArrayReader();
+ assertReadWriteWithSkip(writer, reader, randvalues);
+ }
+
+ @Test
public void testLengths() throws IOException {
DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
ValuesReader reader = new DeltaBinaryPackingValuesReader();
@@ -81,6 +89,16 @@ public class TestDeltaByteArray {
}
}
+ private void assertReadWriteWithSkip(DeltaByteArrayWriter writer, DeltaByteArrayReader reader, String[] vals) throws Exception {
+ Utils.writeData(writer, vals);
+
+ reader.initFromPage(vals.length, writer.getBytes().toByteBuffer(), 0);
+ for (int i = 0; i < vals.length; i += 2) {
+ Assert.assertEquals(Binary.fromString(vals[i]), reader.readBytes());
+ reader.skip();
+ }
+ }
+
@Test
public void testWriterReset() throws Exception {
DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());