You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by bl...@apache.org on 2017/01/19 01:27:35 UTC
[24/50] [abbrv] parquet-mr git commit: PARQUET-623: Fix
DeltaByteArrayReader#skip.
PARQUET-623: Fix DeltaByteArrayReader#skip.
Previously, this passed the skip to the underlying readers, but would
not update previous and would corrupt values or cause exceptions.
Author: Ryan Blue <bl...@apache.org>
Closes #366 from rdblue/PARQUET-623-fix-delta-byte-array-skip and squashes the following commits:
f85800c [Ryan Blue] PARQUET-623: Fix DeltaByteArrayReader#skip.
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/6e95d450
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/6e95d450
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/6e95d450
Branch: refs/heads/parquet-1.8.x
Commit: 6e95d45009848a402aed665b3d92bd2ee6a87c3b
Parents: 71e43ad
Author: Ryan Blue <bl...@apache.org>
Authored: Thu Sep 8 14:22:30 2016 -0700
Committer: Ryan Blue <bl...@apache.org>
Committed: Mon Jan 9 16:54:54 2017 -0800
----------------------------------------------------------------------
.../values/deltastrings/DeltaByteArrayReader.java | 4 ++--
.../values/deltastrings/TestDeltaByteArray.java | 18 ++++++++++++++++++
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/6e95d450/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
index 87ec08e..14fa1ee 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
@@ -54,8 +54,8 @@ public class DeltaByteArrayReader extends ValuesReader implements RequiresPrevio
@Override
public void skip() {
- prefixLengthReader.skip();
- suffixReader.skip();
+ // read the next value to skip so that previous is correct.
+ readBytes();
}
@Override
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/6e95d450/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
index b62ef84..263fce4 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
@@ -19,6 +19,7 @@
package org.apache.parquet.column.values.deltastrings;
import java.io.IOException;
+import java.nio.ByteBuffer;
import org.junit.Test;
import org.junit.Assert;
@@ -49,6 +50,13 @@ public class TestDeltaByteArray {
}
@Test
+ public void testRandomStringsWithSkip() throws Exception {
+ DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024);
+ DeltaByteArrayReader reader = new DeltaByteArrayReader();
+ assertReadWriteWithSkip(writer, reader, randvalues);
+ }
+
+ @Test
public void testLengths() throws IOException {
DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024);
ValuesReader reader = new DeltaBinaryPackingValuesReader();
@@ -80,6 +88,16 @@ public class TestDeltaByteArray {
}
}
+ private void assertReadWriteWithSkip(DeltaByteArrayWriter writer, DeltaByteArrayReader reader, String[] vals) throws Exception {
+ Utils.writeData(writer, vals);
+
+ reader.initFromPage(vals.length, writer.getBytes().toByteArray(), 0);
+ for (int i = 0; i < vals.length; i += 2) {
+ Assert.assertEquals(Binary.fromString(vals[i]), reader.readBytes());
+ reader.skip();
+ }
+ }
+
@Test
public void testWriterReset() throws Exception {
DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024);