You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2018/05/22 22:03:51 UTC
orc git commit: ORC-367: Fix incorrect reads of boolean columns after
a seek.
Repository: orc
Updated Branches:
refs/heads/master 5120baaef -> b0504f55e
ORC-367: Fix incorrect reads of boolean columns after a seek.
Fix #272
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/b0504f55
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/b0504f55
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/b0504f55
Branch: refs/heads/master
Commit: b0504f55e4a68de0c73b89ab5bccb2a673efe9b4
Parents: 5120baa
Author: Owen O'Malley <om...@apache.org>
Authored: Tue May 22 14:43:06 2018 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue May 22 14:45:30 2018 -0700
----------------------------------------------------------------------
.../org/apache/orc/impl/BitFieldReader.java | 8 ++---
.../org/apache/orc/impl/TestBitFieldReader.java | 35 ++++++++++++++++++++
2 files changed, 38 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/b0504f55/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/BitFieldReader.java b/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
index 9102555..5daa204 100644
--- a/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
+++ b/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
@@ -24,9 +24,8 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
public final class BitFieldReader {
private final RunLengthByteReader input;
- /** The number of bits in one item. Non-test code always uses 1. */
private int current;
- private byte currentIdx;
+ private byte currentIdx = 8;
public BitFieldReader(InStream input) {
this.input = new RunLengthByteReader(input);
@@ -42,8 +41,7 @@ public final class BitFieldReader {
}
public int next() throws IOException {
- // mod 8
- if ((currentIdx & 7) == 0) {
+ if (currentIdx > 7) {
readByte();
}
@@ -86,7 +84,7 @@ public final class BitFieldReader {
readByte();
currentIdx = (byte) consumed;
} else {
- currentIdx = 0;
+ currentIdx = 8;
}
}
http://git-wip-us.apache.org/repos/asf/orc/blob/b0504f55/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java b/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
index 156f10a..f7a2a5c 100644
--- a/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
+++ b/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
@@ -111,4 +111,39 @@ public class TestBitFieldReader {
in.skip(0);
}
}
+
+ @Test
+ public void testSeekSkip() throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ BitFieldWriter out = new BitFieldWriter(
+ new OutStream("test", 100, null, collect), 1);
+ final int COUNT = 256;
+ TestInStream.PositionCollector posn = new TestInStream.PositionCollector();
+ for(int i=0; i < COUNT; ++i) {
+ if (i == 200) {
+ out.getPosition(posn);
+ }
+ if (i < COUNT/2) {
+ out.write(i & 1);
+ } else {
+ out.write((i/3) & 1);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ BitFieldReader in = new BitFieldReader(InStream.create("test", new ByteBuffer[]{inBuf},
+ new long[]{0}, inBuf.remaining(), null, 100));
+ in.seek(posn);
+ in.skip(10);
+ for(int r = 210; r < COUNT; ++r) {
+ int x = (int) in.next();
+ if (r < COUNT/2) {
+ assertEquals(r & 1, x);
+ } else {
+ assertEquals((r/3) & 1, x);
+ }
+ }
+ }
}