You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2015/12/12 00:27:58 UTC
[01/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Repository: hive
Updated Branches:
refs/heads/master 3e3d966f3 -> 9c7a78ee3
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestMemoryManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestMemoryManager.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestMemoryManager.java
deleted file mode 100644
index 19aaff3..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestMemoryManager.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
-import org.junit.Test;
-
-import java.lang.management.ManagementFactory;
-
-import static junit.framework.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doubleThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-/**
- * Test the ORC memory manager.
- */
-public class TestMemoryManager {
- private static final double ERROR = 0.000001;
-
- private static class NullCallback implements MemoryManager.Callback {
- public boolean checkMemory(double newScale) {
- return false;
- }
- }
-
- @Test
- public void testBasics() throws Exception {
- Configuration conf = new Configuration();
- MemoryManager mgr = new MemoryManager(conf);
- NullCallback callback = new NullCallback();
- long poolSize = mgr.getTotalMemoryPool();
- assertEquals(Math.round(ManagementFactory.getMemoryMXBean().
- getHeapMemoryUsage().getMax() * 0.5d), poolSize);
- assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
- mgr.addWriter(new Path("p1"), 1000, callback);
- assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
- mgr.addWriter(new Path("p1"), poolSize / 2, callback);
- assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
- mgr.addWriter(new Path("p2"), poolSize / 2, callback);
- assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
- mgr.addWriter(new Path("p3"), poolSize / 2, callback);
- assertEquals(0.6666667, mgr.getAllocationScale(), 0.00001);
- mgr.addWriter(new Path("p4"), poolSize / 2, callback);
- assertEquals(0.5, mgr.getAllocationScale(), 0.000001);
- mgr.addWriter(new Path("p4"), 3 * poolSize / 2, callback);
- assertEquals(0.3333333, mgr.getAllocationScale(), 0.000001);
- mgr.removeWriter(new Path("p1"));
- mgr.removeWriter(new Path("p2"));
- assertEquals(0.5, mgr.getAllocationScale(), 0.00001);
- mgr.removeWriter(new Path("p4"));
- assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
- }
-
- @Test
- public void testConfig() throws Exception {
- Configuration conf = new Configuration();
- conf.set("hive.exec.orc.memory.pool", "0.9");
- MemoryManager mgr = new MemoryManager(conf);
- long mem =
- ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
- System.err.print("Memory = " + mem);
- long pool = mgr.getTotalMemoryPool();
- assertTrue("Pool too small: " + pool, mem * 0.899 < pool);
- assertTrue("Pool too big: " + pool, pool < mem * 0.901);
- }
-
- private static class DoubleMatcher extends BaseMatcher<Double> {
- final double expected;
- final double error;
- DoubleMatcher(double expected, double error) {
- this.expected = expected;
- this.error = error;
- }
-
- @Override
- public boolean matches(Object val) {
- double dbl = (Double) val;
- return Math.abs(dbl - expected) <= error;
- }
-
- @Override
- public void describeTo(Description description) {
- description.appendText("not sufficiently close to ");
- description.appendText(Double.toString(expected));
- }
- }
-
- private static DoubleMatcher closeTo(double value, double error) {
- return new DoubleMatcher(value, error);
- }
-
- @Test
- public void testCallback() throws Exception {
- Configuration conf = new Configuration();
- MemoryManager mgr = new MemoryManager(conf);
- long pool = mgr.getTotalMemoryPool();
- MemoryManager.Callback[] calls = new MemoryManager.Callback[20];
- for(int i=0; i < calls.length; ++i) {
- calls[i] = mock(MemoryManager.Callback.class);
- mgr.addWriter(new Path(Integer.toString(i)), pool/4, calls[i]);
- }
- // add enough rows to get the memory manager to check the limits
- for(int i=0; i < 10000; ++i) {
- mgr.addedRow(1);
- }
- for(int call=0; call < calls.length; ++call) {
- verify(calls[call], times(2))
- .checkMemory(doubleThat(closeTo(0.2, ERROR)));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
index 0f606a4..2fd13c7 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
@@ -29,12 +29,12 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.EncodingStrategy;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
+import org.apache.orc.CompressionKind;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -49,16 +49,16 @@ import com.google.common.primitives.Longs;
@RunWith(value = Parameterized.class)
public class TestNewIntegerEncoding {
- private EncodingStrategy encodingStrategy;
+ private OrcFile.EncodingStrategy encodingStrategy;
- public TestNewIntegerEncoding(EncodingStrategy es) {
+ public TestNewIntegerEncoding( OrcFile.EncodingStrategy es) {
this.encodingStrategy = es;
}
@Parameters
public static Collection<Object[]> data() {
- Object[][] data = new Object[][] { { EncodingStrategy.COMPRESSION },
- { EncodingStrategy.SPEED } };
+ Object[][] data = new Object[][] { { OrcFile.EncodingStrategy.COMPRESSION },
+ { OrcFile.EncodingStrategy.SPEED } };
return Arrays.asList(data);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
index 2992f3c..ebe3096 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
@@ -38,12 +38,12 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
+import com.google.common.primitives.Longs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.Version;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
@@ -77,6 +77,21 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hive.common.util.HiveTestUtils;
+import org.apache.orc.BinaryColumnStatistics;
+import org.apache.orc.BooleanColumnStatistics;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.DecimalColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.OrcProto;
+
+import org.apache.orc.OrcUtils;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
+import org.apache.orc.TypeDescription;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -1794,7 +1809,7 @@ public class TestOrcFile {
}
@Override
- void addWriter(Path path, long requestedAllocation,
+ public void addWriter(Path path, long requestedAllocation,
MemoryManager.Callback callback) {
this.path = path;
this.lastAllocation = requestedAllocation;
@@ -1802,23 +1817,23 @@ public class TestOrcFile {
}
@Override
- synchronized void removeWriter(Path path) {
+ public synchronized void removeWriter(Path path) {
this.path = null;
this.lastAllocation = 0;
}
@Override
- long getTotalMemoryPool() {
+ public long getTotalMemoryPool() {
return totalSpace;
}
@Override
- double getAllocationScale() {
+ public double getAllocationScale() {
return rate;
}
@Override
- void addedRow(int count) throws IOException {
+ public void addedRow(int count) throws IOException {
rows += count;
if (rows % 100 == 0) {
callback.checkMemory(rate);
@@ -1843,7 +1858,7 @@ public class TestOrcFile {
.bufferSize(100)
.rowIndexStride(0)
.memory(memory)
- .version(Version.V_0_11));
+ .version(OrcFile.Version.V_0_11));
assertEquals(testFilePath, memory.path);
for(int i=0; i < 2500; ++i) {
writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
@@ -1879,7 +1894,7 @@ public class TestOrcFile {
.bufferSize(100)
.rowIndexStride(0)
.memory(memory)
- .version(Version.V_0_12));
+ .version(OrcFile.Version.V_0_12));
assertEquals(testFilePath, memory.path);
for(int i=0; i < 2500; ++i) {
writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
@@ -1986,4 +2001,36 @@ public class TestOrcFile {
assertTrue(!rows.hasNext());
assertEquals(3500, rows.getRowNumber());
}
+
+ @Test
+ public void testBitPack64Large() throws Exception {
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+
+ int size = 1080832;
+ long[] inp = new long[size];
+ Random rand = new Random(1234);
+ for (int i = 0; i < size; i++) {
+ inp[i] = rand.nextLong();
+ }
+ List<Long> input = Lists.newArrayList(Longs.asList(inp));
+
+ Writer writer = OrcFile.createWriter(testFilePath,
+ OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.ZLIB));
+ for (Long l : input) {
+ writer.addRow(l);
+ }
+ writer.close();
+
+ Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows();
+ int idx = 0;
+ while (rows.hasNext()) {
+ Object row = rows.next(null);
+ Assert.assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
index 340951b..e96c809 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
@@ -34,6 +34,13 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.IntWritable;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.OrcProto;
+
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
index 032ac4b..966621c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.hive.ql.io.orc;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.StripeInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -47,6 +50,8 @@ import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.orc.OrcProto;
+
import org.junit.Test;
import org.mockito.MockSettings;
import org.mockito.Mockito;
@@ -850,7 +855,7 @@ public class TestOrcRawRecordMerger {
int rowsAddedSinceCheck = 0;
@Override
- synchronized void addedRow(int rows) throws IOException {
+ public synchronized void addedRow(int rows) throws IOException {
rowsAddedSinceCheck += rows;
if (rowsAddedSinceCheck >= 2) {
notifyWriters();
@@ -951,7 +956,7 @@ public class TestOrcRawRecordMerger {
int rowsAddedSinceCheck = 0;
@Override
- synchronized void addedRow(int rows) throws IOException {
+ public synchronized void addedRow(int rows) throws IOException {
rowsAddedSinceCheck += rows;
if (rowsAddedSinceCheck >= 2) {
notifyWriters();
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java
index dde653f..575a948 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSerDeStats.java
@@ -43,6 +43,13 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspe
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hive.common.util.HiveTestUtils;
+import org.apache.orc.BinaryColumnStatistics;
+import org.apache.orc.BooleanColumnStatistics;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
index 4b6af07..a409be8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
@@ -42,10 +42,13 @@ import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.Location;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
import org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl;
-import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.impl.ColumnStatisticsImpl;
+import org.apache.orc.OrcProto;
+
import org.junit.Test;
import org.mockito.MockSettings;
import org.mockito.Mockito;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java
deleted file mode 100644
index 90541f0..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-
-public class TestRunLengthByteReader {
-
- @Test
- public void testUncompressedSeek() throws Exception {
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- RunLengthByteWriter out = new RunLengthByteWriter(new OutStream("test", 100,
- null, collect));
- TestInStream.PositionCollector[] positions =
- new TestInStream.PositionCollector[2048];
- for(int i=0; i < 2048; ++i) {
- positions[i] = new TestInStream.PositionCollector();
- out.getPosition(positions[i]);
- if (i < 1024) {
- out.write((byte) (i/4));
- } else {
- out.write((byte) i);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
- new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), null, 100));
- for(int i=0; i < 2048; ++i) {
- int x = in.next() & 0xff;
- if (i < 1024) {
- assertEquals((i/4) & 0xff, x);
- } else {
- assertEquals(i & 0xff, x);
- }
- }
- for(int i=2047; i >= 0; --i) {
- in.seek(positions[i]);
- int x = in.next() & 0xff;
- if (i < 1024) {
- assertEquals((i/4) & 0xff, x);
- } else {
- assertEquals(i & 0xff, x);
- }
- }
- }
-
- @Test
- public void testCompressedSeek() throws Exception {
- CompressionCodec codec = new SnappyCodec();
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- RunLengthByteWriter out = new RunLengthByteWriter(new OutStream("test", 500,
- codec, collect));
- TestInStream.PositionCollector[] positions =
- new TestInStream.PositionCollector[2048];
- for(int i=0; i < 2048; ++i) {
- positions[i] = new TestInStream.PositionCollector();
- out.getPosition(positions[i]);
- if (i < 1024) {
- out.write((byte) (i/4));
- } else {
- out.write((byte) i);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
- new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), codec, 500));
- for(int i=0; i < 2048; ++i) {
- int x = in.next() & 0xff;
- if (i < 1024) {
- assertEquals((i/4) & 0xff, x);
- } else {
- assertEquals(i & 0xff, x);
- }
- }
- for(int i=2047; i >= 0; --i) {
- in.seek(positions[i]);
- int x = in.next() & 0xff;
- if (i < 1024) {
- assertEquals((i/4) & 0xff, x);
- } else {
- assertEquals(i & 0xff, x);
- }
- }
- }
-
- @Test
- public void testSkips() throws Exception {
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- RunLengthByteWriter out = new RunLengthByteWriter(new OutStream("test", 100,
- null, collect));
- for(int i=0; i < 2048; ++i) {
- if (i < 1024) {
- out.write((byte) (i/16));
- } else {
- out.write((byte) i);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
- new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), null, 100));
- for(int i=0; i < 2048; i += 10) {
- int x = in.next() & 0xff;
- if (i < 1024) {
- assertEquals((i/16) & 0xff, x);
- } else {
- assertEquals(i & 0xff, x);
- }
- if (i < 2038) {
- in.skip(9);
- }
- in.skip(0);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java
deleted file mode 100644
index 562b489..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-import org.junit.Test;
-
-public class TestRunLengthIntegerReader {
-
- public void runSeekTest(CompressionCodec codec) throws Exception {
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- RunLengthIntegerWriter out = new RunLengthIntegerWriter(
- new OutStream("test", 1000, codec, collect), true);
- TestInStream.PositionCollector[] positions =
- new TestInStream.PositionCollector[4096];
- Random random = new Random(99);
- int[] junk = new int[2048];
- for(int i=0; i < junk.length; ++i) {
- junk[i] = random.nextInt();
- }
- for(int i=0; i < 4096; ++i) {
- positions[i] = new TestInStream.PositionCollector();
- out.getPosition(positions[i]);
- // test runs, incrementing runs, non-runs
- if (i < 1024) {
- out.write(i/4);
- } else if (i < 2048) {
- out.write(2*i);
- } else {
- out.write(junk[i-2048]);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- RunLengthIntegerReader in = new RunLengthIntegerReader(InStream.create
- ("test", new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
- codec, 1000), true);
- for(int i=0; i < 2048; ++i) {
- int x = (int) in.next();
- if (i < 1024) {
- assertEquals(i/4, x);
- } else if (i < 2048) {
- assertEquals(2*i, x);
- } else {
- assertEquals(junk[i-2048], x);
- }
- }
- for(int i=2047; i >= 0; --i) {
- in.seek(positions[i]);
- int x = (int) in.next();
- if (i < 1024) {
- assertEquals(i/4, x);
- } else if (i < 2048) {
- assertEquals(2*i, x);
- } else {
- assertEquals(junk[i-2048], x);
- }
- }
- }
-
- @Test
- public void testUncompressedSeek() throws Exception {
- runSeekTest(null);
- }
-
- @Test
- public void testCompressedSeek() throws Exception {
- runSeekTest(new ZlibCodec());
- }
-
- @Test
- public void testSkips() throws Exception {
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- RunLengthIntegerWriter out = new RunLengthIntegerWriter(
- new OutStream("test", 100, null, collect), true);
- for(int i=0; i < 2048; ++i) {
- if (i < 1024) {
- out.write(i);
- } else {
- out.write(256 * i);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- RunLengthIntegerReader in = new RunLengthIntegerReader(InStream.create
- ("test", new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
- null, 100), true);
- for(int i=0; i < 2048; i += 10) {
- int x = (int) in.next();
- if (i < 1024) {
- assertEquals(i, x);
- } else {
- assertEquals(256 * i, x);
- }
- if (i < 2038) {
- in.skip(9);
- }
- in.skip(0);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
deleted file mode 100644
index b3f9cf1..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
-import java.math.BigInteger;
-
-import org.junit.Test;
-
-import com.google.common.math.LongMath;
-
-public class TestSerializationUtils {
-
- private InputStream fromBuffer(ByteArrayOutputStream buffer) {
- return new ByteArrayInputStream(buffer.toByteArray());
- }
-
- @Test
- public void testDoubles() throws Exception {
- double tolerance = 0.0000000000000001;
- ByteArrayOutputStream buffer = new ByteArrayOutputStream();
- SerializationUtils utils = new SerializationUtils();
- utils.writeDouble(buffer, 1343822337.759);
- assertEquals(1343822337.759, utils.readDouble(fromBuffer(buffer)), tolerance);
- buffer = new ByteArrayOutputStream();
- utils.writeDouble(buffer, 0.8);
- double got = utils.readDouble(fromBuffer(buffer));
- assertEquals(0.8, got, tolerance);
- }
-
- @Test
- public void testBigIntegers() throws Exception {
- ByteArrayOutputStream buffer = new ByteArrayOutputStream();
- SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(0));
- assertArrayEquals(new byte[]{0}, buffer.toByteArray());
- assertEquals(0L,
- SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
- buffer.reset();
- SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(1));
- assertArrayEquals(new byte[]{2}, buffer.toByteArray());
- assertEquals(1L,
- SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
- buffer.reset();
- SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(-1));
- assertArrayEquals(new byte[]{1}, buffer.toByteArray());
- assertEquals(-1L,
- SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
- buffer.reset();
- SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(50));
- assertArrayEquals(new byte[]{100}, buffer.toByteArray());
- assertEquals(50L,
- SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
- buffer.reset();
- SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(-50));
- assertArrayEquals(new byte[]{99}, buffer.toByteArray());
- assertEquals(-50L,
- SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
- for(int i=-8192; i < 8192; ++i) {
- buffer.reset();
- SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(i));
- assertEquals("compare length for " + i,
- i >= -64 && i < 64 ? 1 : 2, buffer.size());
- assertEquals("compare result for " + i,
- i, SerializationUtils.readBigInteger(fromBuffer(buffer)).intValue());
- }
- buffer.reset();
- SerializationUtils.writeBigInteger(buffer,
- new BigInteger("123456789abcdef0",16));
- assertEquals(new BigInteger("123456789abcdef0",16),
- SerializationUtils.readBigInteger(fromBuffer(buffer)));
- buffer.reset();
- SerializationUtils.writeBigInteger(buffer,
- new BigInteger("-123456789abcdef0",16));
- assertEquals(new BigInteger("-123456789abcdef0",16),
- SerializationUtils.readBigInteger(fromBuffer(buffer)));
- StringBuilder buf = new StringBuilder();
- for(int i=0; i < 256; ++i) {
- String num = Integer.toHexString(i);
- if (num.length() == 1) {
- buf.append('0');
- }
- buf.append(num);
- }
- buffer.reset();
- SerializationUtils.writeBigInteger(buffer,
- new BigInteger(buf.toString(),16));
- assertEquals(new BigInteger(buf.toString(),16),
- SerializationUtils.readBigInteger(fromBuffer(buffer)));
- buffer.reset();
- SerializationUtils.writeBigInteger(buffer,
- new BigInteger("ff000000000000000000000000000000000000000000ff",16));
- assertEquals(
- new BigInteger("ff000000000000000000000000000000000000000000ff",16),
- SerializationUtils.readBigInteger(fromBuffer(buffer)));
- }
-
- @Test
- public void testSubtractionOverflow() {
- // cross check results with Guava results below
- SerializationUtils utils = new SerializationUtils();
- assertEquals(false, utils.isSafeSubtract(22222222222L, Long.MIN_VALUE));
- assertEquals(false, utils.isSafeSubtract(-22222222222L, Long.MAX_VALUE));
- assertEquals(false, utils.isSafeSubtract(Long.MIN_VALUE, Long.MAX_VALUE));
- assertEquals(true, utils.isSafeSubtract(-1553103058346370095L, 6553103058346370095L));
- assertEquals(true, utils.isSafeSubtract(0, Long.MAX_VALUE));
- assertEquals(true, utils.isSafeSubtract(Long.MIN_VALUE, 0));
- }
-
- @Test
- public void testSubtractionOverflowGuava() {
- try {
- LongMath.checkedSubtract(22222222222L, Long.MIN_VALUE);
- fail("expected ArithmeticException for overflow");
- } catch (ArithmeticException ex) {
- assertEquals(ex.getMessage(), "overflow");
- }
-
- try {
- LongMath.checkedSubtract(-22222222222L, Long.MAX_VALUE);
- fail("expected ArithmeticException for overflow");
- } catch (ArithmeticException ex) {
- assertEquals(ex.getMessage(), "overflow");
- }
-
- try {
- LongMath.checkedSubtract(Long.MIN_VALUE, Long.MAX_VALUE);
- fail("expected ArithmeticException for overflow");
- } catch (ArithmeticException ex) {
- assertEquals(ex.getMessage(), "overflow");
- }
-
- assertEquals(-8106206116692740190L,
- LongMath.checkedSubtract(-1553103058346370095L, 6553103058346370095L));
- assertEquals(-Long.MAX_VALUE, LongMath.checkedSubtract(0, Long.MAX_VALUE));
- assertEquals(Long.MIN_VALUE, LongMath.checkedSubtract(Long.MIN_VALUE, 0));
- }
-
- public static void main(String[] args) throws Exception {
- TestSerializationUtils test = new TestSerializationUtils();
- test.testDoubles();
- test.testBigIntegers();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
index f4d91ea..dfccd9a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.io.orc;
+import org.apache.orc.OrcProto;
+import org.apache.orc.impl.StreamName;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
index 7dca820..41a211b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
@@ -26,10 +26,13 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.Version;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.io.Text;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcProto;
+
+import org.apache.orc.StripeInformation;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -228,7 +231,7 @@ public class TestStringDictionary {
Writer writer = OrcFile.createWriter(
testFilePath,
OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.NONE)
- .version(Version.V_0_11).bufferSize(10000));
+ .version(OrcFile.Version.V_0_11).bufferSize(10000));
for (int i = 0; i < 20000; i++) {
writer.addRow(new Text(String.valueOf(i)));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
deleted file mode 100644
index 5581ad3..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-
-import static junit.framework.Assert.assertEquals;
-
-/**
- * Test the red-black tree with string keys.
- */
-public class TestStringRedBlackTree {
-
- /**
- * Checks the red-black tree rules to make sure that we have correctly built
- * a valid tree.
- *
- * Properties:
- * 1. Red nodes must have black children
- * 2. Each node must have the same black height on both sides.
- *
- * @param node The id of the root of the subtree to check for the red-black
- * tree properties.
- * @return The black-height of the subtree.
- */
- private int checkSubtree(RedBlackTree tree, int node, IntWritable count
- ) throws IOException {
- if (node == RedBlackTree.NULL) {
- return 1;
- }
- count.set(count.get() + 1);
- boolean is_red = tree.isRed(node);
- int left = tree.getLeft(node);
- int right = tree.getRight(node);
- if (is_red) {
- if (tree.isRed(left)) {
- printTree(tree, "", tree.root);
- throw new IllegalStateException("Left node of " + node + " is " + left +
- " and both are red.");
- }
- if (tree.isRed(right)) {
- printTree(tree, "", tree.root);
- throw new IllegalStateException("Right node of " + node + " is " +
- right + " and both are red.");
- }
- }
- int left_depth = checkSubtree(tree, left, count);
- int right_depth = checkSubtree(tree, right, count);
- if (left_depth != right_depth) {
- printTree(tree, "", tree.root);
- throw new IllegalStateException("Lopsided tree at node " + node +
- " with depths " + left_depth + " and " + right_depth);
- }
- if (is_red) {
- return left_depth;
- } else {
- return left_depth + 1;
- }
- }
-
- /**
- * Checks the validity of the entire tree. Also ensures that the number of
- * nodes visited is the same as the size of the set.
- */
- void checkTree(RedBlackTree tree) throws IOException {
- IntWritable count = new IntWritable(0);
- if (tree.isRed(tree.root)) {
- printTree(tree, "", tree.root);
- throw new IllegalStateException("root is red");
- }
- checkSubtree(tree, tree.root, count);
- if (count.get() != tree.size) {
- printTree(tree, "", tree.root);
- throw new IllegalStateException("Broken tree! visited= " + count.get() +
- " size=" + tree.size);
- }
- }
-
- void printTree(RedBlackTree tree, String indent, int node
- ) throws IOException {
- if (node == RedBlackTree.NULL) {
- System.err.println(indent + "NULL");
- } else {
- System.err.println(indent + "Node " + node + " color " +
- (tree.isRed(node) ? "red" : "black"));
- printTree(tree, indent + " ", tree.getLeft(node));
- printTree(tree, indent + " ", tree.getRight(node));
- }
- }
-
- private static class MyVisitor implements StringRedBlackTree.Visitor {
- private final String[] words;
- private final int[] order;
- private final DataOutputBuffer buffer = new DataOutputBuffer();
- int current = 0;
-
- MyVisitor(String[] args, int[] order) {
- words = args;
- this.order = order;
- }
-
- @Override
- public void visit(StringRedBlackTree.VisitorContext context
- ) throws IOException {
- String word = context.getText().toString();
- assertEquals("in word " + current, words[current], word);
- assertEquals("in word " + current, order[current],
- context.getOriginalPosition());
- buffer.reset();
- context.writeBytes(buffer);
- assertEquals(word, new String(buffer.getData(),0,buffer.getLength()));
- current += 1;
- }
- }
-
- void checkContents(StringRedBlackTree tree, int[] order,
- String... params
- ) throws IOException {
- tree.visit(new MyVisitor(params, order));
- }
-
- StringRedBlackTree buildTree(String... params) throws IOException {
- StringRedBlackTree result = new StringRedBlackTree(1000);
- for(String word: params) {
- result.add(word);
- checkTree(result);
- }
- return result;
- }
-
- @Test
- public void test1() throws Exception {
- StringRedBlackTree tree = new StringRedBlackTree(5);
- assertEquals(0, tree.getSizeInBytes());
- checkTree(tree);
- assertEquals(0, tree.add("owen"));
- checkTree(tree);
- assertEquals(1, tree.add("ashutosh"));
- checkTree(tree);
- assertEquals(0, tree.add("owen"));
- checkTree(tree);
- assertEquals(2, tree.add("alan"));
- checkTree(tree);
- assertEquals(2, tree.add("alan"));
- checkTree(tree);
- assertEquals(1, tree.add("ashutosh"));
- checkTree(tree);
- assertEquals(3, tree.add("greg"));
- checkTree(tree);
- assertEquals(4, tree.add("eric"));
- checkTree(tree);
- assertEquals(5, tree.add("arun"));
- checkTree(tree);
- assertEquals(6, tree.size());
- checkTree(tree);
- assertEquals(6, tree.add("eric14"));
- checkTree(tree);
- assertEquals(7, tree.add("o"));
- checkTree(tree);
- assertEquals(8, tree.add("ziggy"));
- checkTree(tree);
- assertEquals(9, tree.add("z"));
- checkTree(tree);
- checkContents(tree, new int[]{2,5,1,4,6,3,7,0,9,8},
- "alan", "arun", "ashutosh", "eric", "eric14", "greg",
- "o", "owen", "z", "ziggy");
- assertEquals(32888, tree.getSizeInBytes());
- // check that adding greg again bumps the count
- assertEquals(3, tree.add("greg"));
- assertEquals(41, tree.getCharacterSize());
- // add some more strings to test the different branches of the
- // rebalancing
- assertEquals(10, tree.add("zak"));
- checkTree(tree);
- assertEquals(11, tree.add("eric1"));
- checkTree(tree);
- assertEquals(12, tree.add("ash"));
- checkTree(tree);
- assertEquals(13, tree.add("harry"));
- checkTree(tree);
- assertEquals(14, tree.add("john"));
- checkTree(tree);
- tree.clear();
- checkTree(tree);
- assertEquals(0, tree.getSizeInBytes());
- assertEquals(0, tree.getCharacterSize());
- }
-
- @Test
- public void test2() throws Exception {
- StringRedBlackTree tree =
- buildTree("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l",
- "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
- assertEquals(26, tree.size());
- checkContents(tree, new int[]{0,1,2, 3,4,5, 6,7,8, 9,10,11, 12,13,14,
- 15,16,17, 18,19,20, 21,22,23, 24,25},
- "a", "b", "c", "d", "e", "f", "g", "h", "i", "j","k", "l", "m", "n", "o",
- "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
- }
-
- @Test
- public void test3() throws Exception {
- StringRedBlackTree tree =
- buildTree("z", "y", "x", "w", "v", "u", "t", "s", "r", "q", "p", "o", "n",
- "m", "l", "k", "j", "i", "h", "g", "f", "e", "d", "c", "b", "a");
- assertEquals(26, tree.size());
- checkContents(tree, new int[]{25,24,23, 22,21,20, 19,18,17, 16,15,14,
- 13,12,11, 10,9,8, 7,6,5, 4,3,2, 1,0},
- "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
- "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
- }
-
- public static void main(String[] args) throws Exception {
- TestStringRedBlackTree test = new TestStringRedBlackTree();
- test.test1();
- test.test2();
- test.test3();
- TestSerializationUtils serUtils = new TestSerializationUtils();
- serUtils.testDoubles();
- TestDynamicArray test6 = new TestDynamicArray();
- test6.testByteArray();
- test6.testIntArray();
- TestZlib zlib = new TestZlib();
- zlib.testCorrupt();
- zlib.testNoOverflow();
- TestInStream inStreamTest = new TestInStream();
- inStreamTest.testUncompressed();
- inStreamTest.testCompressed();
- inStreamTest.testCorruptStream();
- TestRunLengthByteReader rleByte = new TestRunLengthByteReader();
- rleByte.testUncompressedSeek();
- rleByte.testCompressedSeek();
- rleByte.testSkips();
- TestRunLengthIntegerReader rleInt = new TestRunLengthIntegerReader();
- rleInt.testUncompressedSeek();
- rleInt.testCompressedSeek();
- rleInt.testSkips();
- TestBitFieldReader bit = new TestBitFieldReader();
- bit.testUncompressedSeek();
- bit.testCompressedSeek();
- bit.testBiggerItems();
- bit.testSkips();
- TestOrcFile test1 = new TestOrcFile(false);
- test1.test1();
- test1.emptyFile();
- test1.metaData();
- test1.testUnionAndTimestamp();
- test1.columnProjection();
- test1.testSnappy();
- test1.testWithoutIndex();
- test1.testSeek();
- TestFileDump test2 = new TestFileDump();
- test2.testDump();
- TestStreamName test3 = new TestStreamName();
- test3.test1();
- TestInputOutputFormat test4 = new TestInputOutputFormat();
- test4.testInOutFormat();
- test4.testMROutput();
- test4.testEmptyFile();
- test4.testDefaultTypes();
- TestOrcStruct test5 = new TestOrcStruct();
- test5.testStruct();
- test5.testInspectorFromTypeInfo();
- test5.testUnion();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java
index c3095f7..2a82092 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestTypeDescription.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.orc;
import static org.junit.Assert.assertEquals;
+import org.apache.orc.TypeDescription;
import org.junit.Test;
public class TestTypeDescription {
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java
index 5666cd7..3251731 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestUnrolledBitPack.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.io.LongWritable;
+import org.apache.orc.CompressionKind;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
index 070e2ab..f5b06db 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.Version;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
@@ -67,6 +66,23 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hive.common.util.HiveTestUtils;
+import org.apache.orc.BinaryColumnStatistics;
+import org.apache.orc.BooleanColumnStatistics;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.DecimalColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.impl.MetadataReader;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.OrcProto;
+import org.apache.orc.OrcUtils;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -486,7 +502,7 @@ public class TestVectorOrcFile {
TypeDescription schema = TypeDescription.createTimestamp();
Writer writer = OrcFile.createWriter(testFilePath,
OrcFile.writerOptions(conf).setSchema(schema).stripeSize(100000)
- .bufferSize(10000).version(Version.V_0_11));
+ .bufferSize(10000).version(org.apache.orc.OrcFile.Version.V_0_11));
List<Timestamp> tslist = Lists.newArrayList();
tslist.add(Timestamp.valueOf("2037-01-01 00:00:00.000999"));
tslist.add(Timestamp.valueOf("2003-01-01 00:00:00.000000222"));
@@ -1244,7 +1260,7 @@ public class TestVectorOrcFile {
Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
assertEquals(false, reader.rows().hasNext());
- assertEquals(CompressionKind.NONE, reader.getCompression());
+ assertEquals(CompressionKind.NONE, reader.getCompressionKind());
assertEquals(0, reader.getNumberOfRows());
assertEquals(0, reader.getCompressionSize());
assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
@@ -1743,7 +1759,7 @@ public class TestVectorOrcFile {
RecordReader rows = reader.rows();
// get the row index
MetadataReader meta = ((RecordReaderImpl) rows).getMetadataReader();
- RecordReaderImpl.Index index =
+ OrcIndex index =
meta.readRowIndex(reader.getStripes().get(0), null, null, null, null,
null);
// check the primitive columns to make sure they have the right number of
@@ -1883,7 +1899,7 @@ public class TestVectorOrcFile {
}
@Override
- void addWriter(Path path, long requestedAllocation,
+ public void addWriter(Path path, long requestedAllocation,
Callback callback) {
this.path = path;
this.lastAllocation = requestedAllocation;
@@ -1891,23 +1907,23 @@ public class TestVectorOrcFile {
}
@Override
- synchronized void removeWriter(Path path) {
+ public synchronized void removeWriter(Path path) {
this.path = null;
this.lastAllocation = 0;
}
@Override
- long getTotalMemoryPool() {
+ public long getTotalMemoryPool() {
return totalSpace;
}
@Override
- double getAllocationScale() {
+ public double getAllocationScale() {
return rate;
}
@Override
- void addedRow(int count) throws IOException {
+ public void addedRow(int count) throws IOException {
rows += count;
if (rows % 100 == 0) {
callback.checkMemory(rate);
@@ -1927,7 +1943,7 @@ public class TestVectorOrcFile {
.bufferSize(100)
.rowIndexStride(0)
.memory(memory)
- .version(Version.V_0_11));
+ .version(OrcFile.Version.V_0_11));
assertEquals(testFilePath, memory.path);
VectorizedRowBatch batch = schema.createRowBatch();
batch.size = 1;
@@ -1963,7 +1979,7 @@ public class TestVectorOrcFile {
.bufferSize(100)
.rowIndexStride(0)
.memory(memory)
- .version(Version.V_0_12));
+ .version(OrcFile.Version.V_0_12));
VectorizedRowBatch batch = schema.createRowBatch();
assertEquals(testFilePath, memory.path);
batch.size = 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
index c739aef..a777b1c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java
deleted file mode 100644
index fbb99e3..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.fail;
-
-public class TestZlib {
-
- @Test
- public void testNoOverflow() throws Exception {
- ByteBuffer in = ByteBuffer.allocate(10);
- ByteBuffer out = ByteBuffer.allocate(10);
- in.put(new byte[]{1,2,3,4,5,6,7,10});
- in.flip();
- CompressionCodec codec = new ZlibCodec();
- assertEquals(false, codec.compress(in, out, null));
- }
-
- @Test
- public void testCorrupt() throws Exception {
- ByteBuffer buf = ByteBuffer.allocate(1000);
- buf.put(new byte[]{127,-128,0,99,98,-1});
- buf.flip();
- CompressionCodec codec = new ZlibCodec();
- ByteBuffer out = ByteBuffer.allocate(1000);
- try {
- codec.decompress(buf, out);
- fail();
- } catch (IOException ioe) {
- // EXPECTED
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
deleted file mode 100644
index dd2b1d9..0000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.io;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.sql.Date;
-import java.util.Calendar;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableUtils;
-
-
-/**
- * DateWritable
- * Writable equivalent of java.sql.Date.
- *
- * Dates are of the format
- * YYYY-MM-DD
- *
- */
-public class DateWritable implements WritableComparable<DateWritable> {
-
- private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1);
-
- // Local time zone.
- // Java TimeZone has no mention of thread safety. Use thread local instance to be safe.
- private static final ThreadLocal<TimeZone> LOCAL_TIMEZONE = new ThreadLocal<TimeZone>() {
- @Override
- protected TimeZone initialValue() {
- return Calendar.getInstance().getTimeZone();
- }
- };
-
- // Internal representation is an integer representing day offset from our epoch value 1970-01-01
- private int daysSinceEpoch = 0;
-
- /* Constructors */
- public DateWritable() {
- }
-
- public DateWritable(DateWritable d) {
- set(d);
- }
-
- public DateWritable(Date d) {
- set(d);
- }
-
- public DateWritable(int d) {
- set(d);
- }
-
- /**
- * Set the DateWritable based on the days since epoch date.
- * @param d integer value representing days since epoch date
- */
- public void set(int d) {
- daysSinceEpoch = d;
- }
-
- /**
- * Set the DateWritable based on the year/month/day of the date in the local timezone.
- * @param d Date value
- */
- public void set(Date d) {
- if (d == null) {
- daysSinceEpoch = 0;
- return;
- }
-
- set(dateToDays(d));
- }
-
- public void set(DateWritable d) {
- set(d.daysSinceEpoch);
- }
-
- /**
- *
- * @return Date value corresponding to the date in the local time zone
- */
- public Date get() {
- return new Date(daysToMillis(daysSinceEpoch));
- }
-
- public int getDays() {
- return daysSinceEpoch;
- }
-
- /**
- *
- * @return time in seconds corresponding to this DateWritable
- */
- public long getTimeInSeconds() {
- return get().getTime() / 1000;
- }
-
- public static Date timeToDate(long l) {
- return new Date(l * 1000);
- }
-
- public static long daysToMillis(int d) {
- // Convert from day offset to ms in UTC, then apply local timezone offset.
- long millisUtc = d * MILLIS_PER_DAY;
- long tmp = millisUtc - LOCAL_TIMEZONE.get().getOffset(millisUtc);
- // Between millisUtc and tmp, the time zone offset may have changed due to DST.
- // Look up the offset again.
- return millisUtc - LOCAL_TIMEZONE.get().getOffset(tmp);
- }
-
- public static int millisToDays(long millisLocal) {
- long millisUtc = millisLocal + LOCAL_TIMEZONE.get().getOffset(millisLocal);
- int days;
- if (millisUtc >= 0L) {
- days = (int) (millisUtc / MILLIS_PER_DAY);
- } else {
- days = (int) ((millisUtc - 86399999) / MILLIS_PER_DAY);
- }
- return days;
- }
-
- public static int dateToDays(Date d) {
- // convert to equivalent time in UTC, then get day offset
- long millisLocal = d.getTime();
- return millisToDays(millisLocal);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- daysSinceEpoch = WritableUtils.readVInt(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- WritableUtils.writeVInt(out, daysSinceEpoch);
- }
-
- @Override
- public int compareTo(DateWritable d) {
- return daysSinceEpoch - d.daysSinceEpoch;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof DateWritable)) {
- return false;
- }
- return compareTo((DateWritable) o) == 0;
- }
-
- @Override
- public String toString() {
- return get().toString();
- }
-
- @Override
- public int hashCode() {
- return daysSinceEpoch;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 0b5c092..c03ae35 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1045,15 +1045,6 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
@Override
- public DirectDecompressorShim getDirectDecompressor(DirectCompressionType codec) {
- if(zeroCopy) {
- return ZeroCopyShims.getDirectDecompressor(codec);
- }
- /* not supported */
- return null;
- }
-
- @Override
public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext context) {
return context.getConfiguration();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java
index 8de08ad..6ef0467 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/ZeroCopyShims.java
@@ -23,13 +23,7 @@ import java.util.EnumSet;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.ReadOption;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectCompressionType;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectDecompressorShim;
import org.apache.hadoop.io.ByteBufferPool;
-import org.apache.hadoop.io.compress.DirectDecompressor;
-import org.apache.hadoop.io.compress.snappy.SnappyDecompressor.SnappyDirectDecompressor;
-import org.apache.hadoop.io.compress.zlib.ZlibDecompressor.CompressionHeader;
-import org.apache.hadoop.io.compress.zlib.ZlibDecompressor.ZlibDirectDecompressor;
import org.apache.hadoop.hive.shims.HadoopShims.ByteBufferPoolShim;
import org.apache.hadoop.hive.shims.HadoopShims.ZeroCopyReaderShim;
@@ -89,40 +83,4 @@ class ZeroCopyShims {
return new ZeroCopyAdapter(in, pool);
}
- private static final class DirectDecompressorAdapter implements
- DirectDecompressorShim {
- private final DirectDecompressor decompressor;
-
- public DirectDecompressorAdapter(DirectDecompressor decompressor) {
- this.decompressor = decompressor;
- }
-
- public void decompress(ByteBuffer src, ByteBuffer dst) throws IOException {
- this.decompressor.decompress(src, dst);
- }
- }
-
- public static DirectDecompressorShim getDirectDecompressor(
- DirectCompressionType codec) {
- DirectDecompressor decompressor = null;
- switch (codec) {
- case ZLIB: {
- decompressor = new ZlibDirectDecompressor();
- }
- break;
- case ZLIB_NOHEADER: {
- decompressor = new ZlibDirectDecompressor(CompressionHeader.NO_HEADER, 0);
- }
- break;
- case SNAPPY: {
- decompressor = new SnappyDirectDecompressor();
- }
- break;
- }
- if (decompressor != null) {
- return new DirectDecompressorAdapter(decompressor);
- }
- /* not supported */
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index f9785ec..7a5a9b5 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -488,19 +488,6 @@ public interface HadoopShims {
public void releaseBuffer(ByteBuffer buffer);
}
- public enum DirectCompressionType {
- NONE,
- ZLIB_NOHEADER,
- ZLIB,
- SNAPPY,
- };
-
- public interface DirectDecompressorShim {
- public void decompress(ByteBuffer src, ByteBuffer dst) throws IOException;
- }
-
- public DirectDecompressorShim getDirectDecompressor(DirectCompressionType codec);
-
/**
* Get configuration from JobContext
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java b/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
new file mode 100644
index 0000000..86b838c
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.common.io.DiskRange;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Disk range information class containing disk ranges and total length.
+ */
+public class DiskRangeInfo {
+ List<DiskRange> diskRanges; // TODO: use DiskRangeList instead
+ long totalLength;
+
+ public DiskRangeInfo(int indexBaseOffset) {
+ this.diskRanges = Lists.newArrayList();
+ // Some data is missing from the stream for PPD uncompressed read (because index offset is
+ // relative to the entire stream and we only read part of stream if RGs are filtered; unlike
+ // with compressed data where PPD only filters CBs, so we always get full CB, and index offset
+ // is relative to CB). To take care of the case when UncompressedStream goes seeking around by
+ // its incorrect (relative to partial stream) index offset, we will increase the length by our
+ // offset-relative-to-the-stream, and also account for it in buffers (see createDiskRangeInfo).
+ // So, index offset now works; as long as noone seeks into this data before the RG (why would
+ // they), everything works. This is hacky... Stream shouldn't depend on having all the data.
+ this.totalLength = indexBaseOffset;
+ }
+
+ public void addDiskRange(DiskRange diskRange) {
+ diskRanges.add(diskRange);
+ totalLength += diskRange.getLength();
+ }
+
+ public List<DiskRange> getDiskRanges() {
+ return diskRanges;
+ }
+
+ public long getTotalLength() {
+ return totalLength;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
new file mode 100644
index 0000000..dd2b1d9
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.sql.Date;
+import java.util.Calendar;
+import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableUtils;
+
+
+/**
+ * DateWritable
+ * Writable equivalent of java.sql.Date.
+ *
+ * Dates are of the format
+ * YYYY-MM-DD
+ *
+ */
+public class DateWritable implements WritableComparable<DateWritable> {
+
+ private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1);
+
+ // Local time zone.
+ // Java TimeZone has no mention of thread safety. Use thread local instance to be safe.
+ private static final ThreadLocal<TimeZone> LOCAL_TIMEZONE = new ThreadLocal<TimeZone>() {
+ @Override
+ protected TimeZone initialValue() {
+ return Calendar.getInstance().getTimeZone();
+ }
+ };
+
+ // Internal representation is an integer representing day offset from our epoch value 1970-01-01
+ private int daysSinceEpoch = 0;
+
+ /* Constructors */
+ public DateWritable() {
+ }
+
+ public DateWritable(DateWritable d) {
+ set(d);
+ }
+
+ public DateWritable(Date d) {
+ set(d);
+ }
+
+ public DateWritable(int d) {
+ set(d);
+ }
+
+ /**
+ * Set the DateWritable based on the days since epoch date.
+ * @param d integer value representing days since epoch date
+ */
+ public void set(int d) {
+ daysSinceEpoch = d;
+ }
+
+ /**
+ * Set the DateWritable based on the year/month/day of the date in the local timezone.
+ * @param d Date value
+ */
+ public void set(Date d) {
+ if (d == null) {
+ daysSinceEpoch = 0;
+ return;
+ }
+
+ set(dateToDays(d));
+ }
+
+ public void set(DateWritable d) {
+ set(d.daysSinceEpoch);
+ }
+
+ /**
+ *
+ * @return Date value corresponding to the date in the local time zone
+ */
+ public Date get() {
+ return new Date(daysToMillis(daysSinceEpoch));
+ }
+
+ public int getDays() {
+ return daysSinceEpoch;
+ }
+
+ /**
+ *
+ * @return time in seconds corresponding to this DateWritable
+ */
+ public long getTimeInSeconds() {
+ return get().getTime() / 1000;
+ }
+
+ public static Date timeToDate(long l) {
+ return new Date(l * 1000);
+ }
+
+ public static long daysToMillis(int d) {
+ // Convert from day offset to ms in UTC, then apply local timezone offset.
+ long millisUtc = d * MILLIS_PER_DAY;
+ long tmp = millisUtc - LOCAL_TIMEZONE.get().getOffset(millisUtc);
+ // Between millisUtc and tmp, the time zone offset may have changed due to DST.
+ // Look up the offset again.
+ return millisUtc - LOCAL_TIMEZONE.get().getOffset(tmp);
+ }
+
+ public static int millisToDays(long millisLocal) {
+ long millisUtc = millisLocal + LOCAL_TIMEZONE.get().getOffset(millisLocal);
+ int days;
+ if (millisUtc >= 0L) {
+ days = (int) (millisUtc / MILLIS_PER_DAY);
+ } else {
+ days = (int) ((millisUtc - 86399999) / MILLIS_PER_DAY);
+ }
+ return days;
+ }
+
+ public static int dateToDays(Date d) {
+ // convert to equivalent time in UTC, then get day offset
+ long millisLocal = d.getTime();
+ return millisToDays(millisLocal);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ daysSinceEpoch = WritableUtils.readVInt(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ WritableUtils.writeVInt(out, daysSinceEpoch);
+ }
+
+ @Override
+ public int compareTo(DateWritable d) {
+ return daysSinceEpoch - d.daysSinceEpoch;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof DateWritable)) {
+ return false;
+ }
+ return compareTo((DateWritable) o) == 0;
+ }
+
+ @Override
+ public String toString() {
+ return get().toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return daysSinceEpoch;
+ }
+}
[08/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
----------------------------------------------------------------------
diff --git a/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java b/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
deleted file mode 100644
index 5202476..0000000
--- a/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
+++ /dev/null
@@ -1,19280 +0,0 @@
-// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: orc_proto.proto
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-public final class OrcProto {
- private OrcProto() {}
- public static void registerAllExtensions(
- com.google.protobuf.ExtensionRegistry registry) {
- }
- /**
- * Protobuf enum {@code orc.proto.CompressionKind}
- */
- public enum CompressionKind
- implements com.google.protobuf.ProtocolMessageEnum {
- /**
- * <code>NONE = 0;</code>
- */
- NONE(0, 0),
- /**
- * <code>ZLIB = 1;</code>
- */
- ZLIB(1, 1),
- /**
- * <code>SNAPPY = 2;</code>
- */
- SNAPPY(2, 2),
- /**
- * <code>LZO = 3;</code>
- */
- LZO(3, 3),
- ;
-
- /**
- * <code>NONE = 0;</code>
- */
- public static final int NONE_VALUE = 0;
- /**
- * <code>ZLIB = 1;</code>
- */
- public static final int ZLIB_VALUE = 1;
- /**
- * <code>SNAPPY = 2;</code>
- */
- public static final int SNAPPY_VALUE = 2;
- /**
- * <code>LZO = 3;</code>
- */
- public static final int LZO_VALUE = 3;
-
-
- public final int getNumber() { return value; }
-
- public static CompressionKind valueOf(int value) {
- switch (value) {
- case 0: return NONE;
- case 1: return ZLIB;
- case 2: return SNAPPY;
- case 3: return LZO;
- default: return null;
- }
- }
-
- public static com.google.protobuf.Internal.EnumLiteMap<CompressionKind>
- internalGetValueMap() {
- return internalValueMap;
- }
- private static com.google.protobuf.Internal.EnumLiteMap<CompressionKind>
- internalValueMap =
- new com.google.protobuf.Internal.EnumLiteMap<CompressionKind>() {
- public CompressionKind findValueByNumber(int number) {
- return CompressionKind.valueOf(number);
- }
- };
-
- public final com.google.protobuf.Descriptors.EnumValueDescriptor
- getValueDescriptor() {
- return getDescriptor().getValues().get(index);
- }
- public final com.google.protobuf.Descriptors.EnumDescriptor
- getDescriptorForType() {
- return getDescriptor();
- }
- public static final com.google.protobuf.Descriptors.EnumDescriptor
- getDescriptor() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.getDescriptor().getEnumTypes().get(0);
- }
-
- private static final CompressionKind[] VALUES = values();
-
- public static CompressionKind valueOf(
- com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
- if (desc.getType() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "EnumValueDescriptor is not for this type.");
- }
- return VALUES[desc.getIndex()];
- }
-
- private final int index;
- private final int value;
-
- private CompressionKind(int index, int value) {
- this.index = index;
- this.value = value;
- }
-
- // @@protoc_insertion_point(enum_scope:orc.proto.CompressionKind)
- }
-
- public interface IntegerStatisticsOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // optional sint64 minimum = 1;
- /**
- * <code>optional sint64 minimum = 1;</code>
- */
- boolean hasMinimum();
- /**
- * <code>optional sint64 minimum = 1;</code>
- */
- long getMinimum();
-
- // optional sint64 maximum = 2;
- /**
- * <code>optional sint64 maximum = 2;</code>
- */
- boolean hasMaximum();
- /**
- * <code>optional sint64 maximum = 2;</code>
- */
- long getMaximum();
-
- // optional sint64 sum = 3;
- /**
- * <code>optional sint64 sum = 3;</code>
- */
- boolean hasSum();
- /**
- * <code>optional sint64 sum = 3;</code>
- */
- long getSum();
- }
- /**
- * Protobuf type {@code orc.proto.IntegerStatistics}
- */
- public static final class IntegerStatistics extends
- com.google.protobuf.GeneratedMessage
- implements IntegerStatisticsOrBuilder {
- // Use IntegerStatistics.newBuilder() to construct.
- private IntegerStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private IntegerStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final IntegerStatistics defaultInstance;
- public static IntegerStatistics getDefaultInstance() {
- return defaultInstance;
- }
-
- public IntegerStatistics getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private IntegerStatistics(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 8: {
- bitField0_ |= 0x00000001;
- minimum_ = input.readSInt64();
- break;
- }
- case 16: {
- bitField0_ |= 0x00000002;
- maximum_ = input.readSInt64();
- break;
- }
- case 24: {
- bitField0_ |= 0x00000004;
- sum_ = input.readSInt64();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.Builder.class);
- }
-
- public static com.google.protobuf.Parser<IntegerStatistics> PARSER =
- new com.google.protobuf.AbstractParser<IntegerStatistics>() {
- public IntegerStatistics parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new IntegerStatistics(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<IntegerStatistics> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // optional sint64 minimum = 1;
- public static final int MINIMUM_FIELD_NUMBER = 1;
- private long minimum_;
- /**
- * <code>optional sint64 minimum = 1;</code>
- */
- public boolean hasMinimum() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional sint64 minimum = 1;</code>
- */
- public long getMinimum() {
- return minimum_;
- }
-
- // optional sint64 maximum = 2;
- public static final int MAXIMUM_FIELD_NUMBER = 2;
- private long maximum_;
- /**
- * <code>optional sint64 maximum = 2;</code>
- */
- public boolean hasMaximum() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional sint64 maximum = 2;</code>
- */
- public long getMaximum() {
- return maximum_;
- }
-
- // optional sint64 sum = 3;
- public static final int SUM_FIELD_NUMBER = 3;
- private long sum_;
- /**
- * <code>optional sint64 sum = 3;</code>
- */
- public boolean hasSum() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>optional sint64 sum = 3;</code>
- */
- public long getSum() {
- return sum_;
- }
-
- private void initFields() {
- minimum_ = 0L;
- maximum_ = 0L;
- sum_ = 0L;
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeSInt64(1, minimum_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeSInt64(2, maximum_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeSInt64(3, sum_);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeSInt64Size(1, minimum_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeSInt64Size(2, maximum_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- size += com.google.protobuf.CodedOutputStream
- .computeSInt64Size(3, sum_);
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code orc.proto.IntegerStatistics}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatisticsOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- minimum_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000001);
- maximum_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000002);
- sum_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000004);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_descriptor;
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics getDefaultInstanceForType() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics build() {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics buildPartial() {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.minimum_ = minimum_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- result.maximum_ = maximum_;
- if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
- to_bitField0_ |= 0x00000004;
- }
- result.sum_ = sum_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics) {
- return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics other) {
- if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance()) return this;
- if (other.hasMinimum()) {
- setMinimum(other.getMinimum());
- }
- if (other.hasMaximum()) {
- setMaximum(other.getMaximum());
- }
- if (other.hasSum()) {
- setSum(other.getSum());
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // optional sint64 minimum = 1;
- private long minimum_ ;
- /**
- * <code>optional sint64 minimum = 1;</code>
- */
- public boolean hasMinimum() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional sint64 minimum = 1;</code>
- */
- public long getMinimum() {
- return minimum_;
- }
- /**
- * <code>optional sint64 minimum = 1;</code>
- */
- public Builder setMinimum(long value) {
- bitField0_ |= 0x00000001;
- minimum_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional sint64 minimum = 1;</code>
- */
- public Builder clearMinimum() {
- bitField0_ = (bitField0_ & ~0x00000001);
- minimum_ = 0L;
- onChanged();
- return this;
- }
-
- // optional sint64 maximum = 2;
- private long maximum_ ;
- /**
- * <code>optional sint64 maximum = 2;</code>
- */
- public boolean hasMaximum() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional sint64 maximum = 2;</code>
- */
- public long getMaximum() {
- return maximum_;
- }
- /**
- * <code>optional sint64 maximum = 2;</code>
- */
- public Builder setMaximum(long value) {
- bitField0_ |= 0x00000002;
- maximum_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional sint64 maximum = 2;</code>
- */
- public Builder clearMaximum() {
- bitField0_ = (bitField0_ & ~0x00000002);
- maximum_ = 0L;
- onChanged();
- return this;
- }
-
- // optional sint64 sum = 3;
- private long sum_ ;
- /**
- * <code>optional sint64 sum = 3;</code>
- */
- public boolean hasSum() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>optional sint64 sum = 3;</code>
- */
- public long getSum() {
- return sum_;
- }
- /**
- * <code>optional sint64 sum = 3;</code>
- */
- public Builder setSum(long value) {
- bitField0_ |= 0x00000004;
- sum_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional sint64 sum = 3;</code>
- */
- public Builder clearSum() {
- bitField0_ = (bitField0_ & ~0x00000004);
- sum_ = 0L;
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:orc.proto.IntegerStatistics)
- }
-
- static {
- defaultInstance = new IntegerStatistics(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:orc.proto.IntegerStatistics)
- }
-
- public interface DoubleStatisticsOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // optional double minimum = 1;
- /**
- * <code>optional double minimum = 1;</code>
- */
- boolean hasMinimum();
- /**
- * <code>optional double minimum = 1;</code>
- */
- double getMinimum();
-
- // optional double maximum = 2;
- /**
- * <code>optional double maximum = 2;</code>
- */
- boolean hasMaximum();
- /**
- * <code>optional double maximum = 2;</code>
- */
- double getMaximum();
-
- // optional double sum = 3;
- /**
- * <code>optional double sum = 3;</code>
- */
- boolean hasSum();
- /**
- * <code>optional double sum = 3;</code>
- */
- double getSum();
- }
- /**
- * Protobuf type {@code orc.proto.DoubleStatistics}
- */
- public static final class DoubleStatistics extends
- com.google.protobuf.GeneratedMessage
- implements DoubleStatisticsOrBuilder {
- // Use DoubleStatistics.newBuilder() to construct.
- private DoubleStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private DoubleStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final DoubleStatistics defaultInstance;
- public static DoubleStatistics getDefaultInstance() {
- return defaultInstance;
- }
-
- public DoubleStatistics getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private DoubleStatistics(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 9: {
- bitField0_ |= 0x00000001;
- minimum_ = input.readDouble();
- break;
- }
- case 17: {
- bitField0_ |= 0x00000002;
- maximum_ = input.readDouble();
- break;
- }
- case 25: {
- bitField0_ |= 0x00000004;
- sum_ = input.readDouble();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.Builder.class);
- }
-
- public static com.google.protobuf.Parser<DoubleStatistics> PARSER =
- new com.google.protobuf.AbstractParser<DoubleStatistics>() {
- public DoubleStatistics parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new DoubleStatistics(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<DoubleStatistics> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // optional double minimum = 1;
- public static final int MINIMUM_FIELD_NUMBER = 1;
- private double minimum_;
- /**
- * <code>optional double minimum = 1;</code>
- */
- public boolean hasMinimum() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional double minimum = 1;</code>
- */
- public double getMinimum() {
- return minimum_;
- }
-
- // optional double maximum = 2;
- public static final int MAXIMUM_FIELD_NUMBER = 2;
- private double maximum_;
- /**
- * <code>optional double maximum = 2;</code>
- */
- public boolean hasMaximum() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional double maximum = 2;</code>
- */
- public double getMaximum() {
- return maximum_;
- }
-
- // optional double sum = 3;
- public static final int SUM_FIELD_NUMBER = 3;
- private double sum_;
- /**
- * <code>optional double sum = 3;</code>
- */
- public boolean hasSum() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>optional double sum = 3;</code>
- */
- public double getSum() {
- return sum_;
- }
-
- private void initFields() {
- minimum_ = 0D;
- maximum_ = 0D;
- sum_ = 0D;
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeDouble(1, minimum_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeDouble(2, maximum_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeDouble(3, sum_);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeDoubleSize(1, minimum_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeDoubleSize(2, maximum_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- size += com.google.protobuf.CodedOutputStream
- .computeDoubleSize(3, sum_);
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code orc.proto.DoubleStatistics}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatisticsOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- minimum_ = 0D;
- bitField0_ = (bitField0_ & ~0x00000001);
- maximum_ = 0D;
- bitField0_ = (bitField0_ & ~0x00000002);
- sum_ = 0D;
- bitField0_ = (bitField0_ & ~0x00000004);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_descriptor;
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics getDefaultInstanceForType() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics build() {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics buildPartial() {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.minimum_ = minimum_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- result.maximum_ = maximum_;
- if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
- to_bitField0_ |= 0x00000004;
- }
- result.sum_ = sum_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics) {
- return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics other) {
- if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDefaultInstance()) return this;
- if (other.hasMinimum()) {
- setMinimum(other.getMinimum());
- }
- if (other.hasMaximum()) {
- setMaximum(other.getMaximum());
- }
- if (other.hasSum()) {
- setSum(other.getSum());
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // optional double minimum = 1;
- private double minimum_ ;
- /**
- * <code>optional double minimum = 1;</code>
- */
- public boolean hasMinimum() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional double minimum = 1;</code>
- */
- public double getMinimum() {
- return minimum_;
- }
- /**
- * <code>optional double minimum = 1;</code>
- */
- public Builder setMinimum(double value) {
- bitField0_ |= 0x00000001;
- minimum_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional double minimum = 1;</code>
- */
- public Builder clearMinimum() {
- bitField0_ = (bitField0_ & ~0x00000001);
- minimum_ = 0D;
- onChanged();
- return this;
- }
-
- // optional double maximum = 2;
- private double maximum_ ;
- /**
- * <code>optional double maximum = 2;</code>
- */
- public boolean hasMaximum() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional double maximum = 2;</code>
- */
- public double getMaximum() {
- return maximum_;
- }
- /**
- * <code>optional double maximum = 2;</code>
- */
- public Builder setMaximum(double value) {
- bitField0_ |= 0x00000002;
- maximum_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional double maximum = 2;</code>
- */
- public Builder clearMaximum() {
- bitField0_ = (bitField0_ & ~0x00000002);
- maximum_ = 0D;
- onChanged();
- return this;
- }
-
- // optional double sum = 3;
- private double sum_ ;
- /**
- * <code>optional double sum = 3;</code>
- */
- public boolean hasSum() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>optional double sum = 3;</code>
- */
- public double getSum() {
- return sum_;
- }
- /**
- * <code>optional double sum = 3;</code>
- */
- public Builder setSum(double value) {
- bitField0_ |= 0x00000004;
- sum_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional double sum = 3;</code>
- */
- public Builder clearSum() {
- bitField0_ = (bitField0_ & ~0x00000004);
- sum_ = 0D;
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:orc.proto.DoubleStatistics)
- }
-
- static {
- defaultInstance = new DoubleStatistics(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:orc.proto.DoubleStatistics)
- }
-
- public interface StringStatisticsOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // optional string minimum = 1;
- /**
- * <code>optional string minimum = 1;</code>
- */
- boolean hasMinimum();
- /**
- * <code>optional string minimum = 1;</code>
- */
- java.lang.String getMinimum();
- /**
- * <code>optional string minimum = 1;</code>
- */
- com.google.protobuf.ByteString
- getMinimumBytes();
-
- // optional string maximum = 2;
- /**
- * <code>optional string maximum = 2;</code>
- */
- boolean hasMaximum();
- /**
- * <code>optional string maximum = 2;</code>
- */
- java.lang.String getMaximum();
- /**
- * <code>optional string maximum = 2;</code>
- */
- com.google.protobuf.ByteString
- getMaximumBytes();
-
- // optional sint64 sum = 3;
- /**
- * <code>optional sint64 sum = 3;</code>
- *
- * <pre>
- * sum will store the total length of all strings in a stripe
- * </pre>
- */
- boolean hasSum();
- /**
- * <code>optional sint64 sum = 3;</code>
- *
- * <pre>
- * sum will store the total length of all strings in a stripe
- * </pre>
- */
- long getSum();
- }
- /**
- * Protobuf type {@code orc.proto.StringStatistics}
- */
- public static final class StringStatistics extends
- com.google.protobuf.GeneratedMessage
- implements StringStatisticsOrBuilder {
- // Use StringStatistics.newBuilder() to construct.
- private StringStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private StringStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final StringStatistics defaultInstance;
- public static StringStatistics getDefaultInstance() {
- return defaultInstance;
- }
-
- public StringStatistics getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private StringStatistics(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- bitField0_ |= 0x00000001;
- minimum_ = input.readBytes();
- break;
- }
- case 18: {
- bitField0_ |= 0x00000002;
- maximum_ = input.readBytes();
- break;
- }
- case 24: {
- bitField0_ |= 0x00000004;
- sum_ = input.readSInt64();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_StringStatistics_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_StringStatistics_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.Builder.class);
- }
-
- public static com.google.protobuf.Parser<StringStatistics> PARSER =
- new com.google.protobuf.AbstractParser<StringStatistics>() {
- public StringStatistics parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new StringStatistics(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<StringStatistics> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // optional string minimum = 1;
- public static final int MINIMUM_FIELD_NUMBER = 1;
- private java.lang.Object minimum_;
- /**
- * <code>optional string minimum = 1;</code>
- */
- public boolean hasMinimum() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional string minimum = 1;</code>
- */
- public java.lang.String getMinimum() {
- java.lang.Object ref = minimum_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- minimum_ = s;
- }
- return s;
- }
- }
- /**
- * <code>optional string minimum = 1;</code>
- */
- public com.google.protobuf.ByteString
- getMinimumBytes() {
- java.lang.Object ref = minimum_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- minimum_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // optional string maximum = 2;
- public static final int MAXIMUM_FIELD_NUMBER = 2;
- private java.lang.Object maximum_;
- /**
- * <code>optional string maximum = 2;</code>
- */
- public boolean hasMaximum() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional string maximum = 2;</code>
- */
- public java.lang.String getMaximum() {
- java.lang.Object ref = maximum_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- maximum_ = s;
- }
- return s;
- }
- }
- /**
- * <code>optional string maximum = 2;</code>
- */
- public com.google.protobuf.ByteString
- getMaximumBytes() {
- java.lang.Object ref = maximum_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- maximum_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // optional sint64 sum = 3;
- public static final int SUM_FIELD_NUMBER = 3;
- private long sum_;
- /**
- * <code>optional sint64 sum = 3;</code>
- *
- * <pre>
- * sum will store the total length of all strings in a stripe
- * </pre>
- */
- public boolean hasSum() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>optional sint64 sum = 3;</code>
- *
- * <pre>
- * sum will store the total length of all strings in a stripe
- * </pre>
- */
- public long getSum() {
- return sum_;
- }
-
- private void initFields() {
- minimum_ = "";
- maximum_ = "";
- sum_ = 0L;
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, getMinimumBytes());
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeBytes(2, getMaximumBytes());
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeSInt64(3, sum_);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, getMinimumBytes());
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(2, getMaximumBytes());
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- size += com.google.protobuf.CodedOutputStream
- .computeSInt64Size(3, sum_);
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code orc.proto.StringStatistics}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatisticsOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_StringStatistics_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_StringStatistics_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- minimum_ = "";
- bitField0_ = (bitField0_ & ~0x00000001);
- maximum_ = "";
- bitField0_ = (bitField0_ & ~0x00000002);
- sum_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000004);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_StringStatistics_descriptor;
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics getDefaultInstanceForType() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics build() {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics buildPartial() {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.minimum_ = minimum_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- result.maximum_ = maximum_;
- if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
- to_bitField0_ |= 0x00000004;
- }
- result.sum_ = sum_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics) {
- return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics other) {
- if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance()) return this;
- if (other.hasMinimum()) {
- bitField0_ |= 0x00000001;
- minimum_ = other.minimum_;
- onChanged();
- }
- if (other.hasMaximum()) {
- bitField0_ |= 0x00000002;
- maximum_ = other.maximum_;
- onChanged();
- }
- if (other.hasSum()) {
- setSum(other.getSum());
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // optional string minimum = 1;
- private java.lang.Object minimum_ = "";
- /**
- * <code>optional string minimum = 1;</code>
- */
- public boolean hasMinimum() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional string minimum = 1;</code>
- */
- public java.lang.String getMinimum() {
- java.lang.Object ref = minimum_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- minimum_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>optional string minimum = 1;</code>
- */
- public com.google.protobuf.ByteString
- getMinimumBytes() {
- java.lang.Object ref = minimum_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- minimum_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>optional string minimum = 1;</code>
- */
- public Builder setMinimum(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- minimum_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional string minimum = 1;</code>
- */
- public Builder clearMinimum() {
- bitField0_ = (bitField0_ & ~0x00000001);
- minimum_ = getDefaultInstance().getMinimum();
- onChanged();
- return this;
- }
- /**
- * <code>optional string minimum = 1;</code>
- */
- public Builder setMinimumBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- minimum_ = value;
- onChanged();
- return this;
- }
-
- // optional string maximum = 2;
- private java.lang.Object maximum_ = "";
- /**
- * <code>optional string maximum = 2;</code>
- */
- public boolean hasMaximum() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional string maximum = 2;</code>
- */
- public java.lang.String getMaximum() {
- java.lang.Object ref = maximum_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- maximum_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>optional string maximum = 2;</code>
- */
- public com.google.protobuf.ByteString
- getMaximumBytes() {
- java.lang.Object ref = maximum_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- maximum_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>optional string maximum = 2;</code>
- */
- public Builder setMaximum(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000002;
- maximum_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional string maximum = 2;</code>
- */
- public Builder clearMaximum() {
- bitField0_ = (bitField0_ & ~0x00000002);
- maximum_ = getDefaultInstance().getMaximum();
- onChanged();
- return this;
- }
- /**
- * <code>optional string maximum = 2;</code>
- */
- public Builder setMaximumBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000002;
- maximum_ = value;
- onChanged();
- return this;
- }
-
- // optional sint64 sum = 3;
- private long sum_ ;
- /**
- * <code>optional sint64 sum = 3;</code>
- *
- * <pre>
- * sum will store the total length of all strings in a stripe
- * </pre>
- */
- public boolean hasSum() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>optional sint64 sum = 3;</code>
- *
- * <pre>
- * sum will store the total length of all strings in a stripe
- * </pre>
- */
- public long getSum() {
- return sum_;
- }
- /**
- * <code>optional sint64 sum = 3;</code>
- *
- * <pre>
- * sum will store the total length of all strings in a stripe
- * </pre>
- */
- public Builder setSum(long value) {
- bitField0_ |= 0x00000004;
- sum_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional sint64 sum = 3;</code>
- *
- * <pre>
- * sum will store the total length of all strings in a stripe
- * </pre>
- */
- public Builder clearSum() {
- bitField0_ = (bitField0_ & ~0x00000004);
- sum_ = 0L;
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:orc.proto.StringStatistics)
- }
-
- static {
- defaultInstance = new StringStatistics(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:orc.proto.StringStatistics)
- }
-
- public interface BucketStatisticsOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // repeated uint64 count = 1 [packed = true];
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- java.util.List<java.lang.Long> getCountList();
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- int getCountCount();
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- long getCount(int index);
- }
- /**
- * Protobuf type {@code orc.proto.BucketStatistics}
- */
- public static final class BucketStatistics extends
- com.google.protobuf.GeneratedMessage
- implements BucketStatisticsOrBuilder {
- // Use BucketStatistics.newBuilder() to construct.
- private BucketStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private BucketStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final BucketStatistics defaultInstance;
- public static BucketStatistics getDefaultInstance() {
- return defaultInstance;
- }
-
- public BucketStatistics getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private BucketStatistics(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 8: {
- if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
- count_ = new java.util.ArrayList<java.lang.Long>();
- mutable_bitField0_ |= 0x00000001;
- }
- count_.add(input.readUInt64());
- break;
- }
- case 10: {
- int length = input.readRawVarint32();
- int limit = input.pushLimit(length);
- if (!((mutable_bitField0_ & 0x00000001) == 0x00000001) && input.getBytesUntilLimit() > 0) {
- count_ = new java.util.ArrayList<java.lang.Long>();
- mutable_bitField0_ |= 0x00000001;
- }
- while (input.getBytesUntilLimit() > 0) {
- count_.add(input.readUInt64());
- }
- input.popLimit(limit);
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
- count_ = java.util.Collections.unmodifiableList(count_);
- }
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_BucketStatistics_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_BucketStatistics_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.Builder.class);
- }
-
- public static com.google.protobuf.Parser<BucketStatistics> PARSER =
- new com.google.protobuf.AbstractParser<BucketStatistics>() {
- public BucketStatistics parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new BucketStatistics(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<BucketStatistics> getParserForType() {
- return PARSER;
- }
-
- // repeated uint64 count = 1 [packed = true];
- public static final int COUNT_FIELD_NUMBER = 1;
- private java.util.List<java.lang.Long> count_;
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public java.util.List<java.lang.Long>
- getCountList() {
- return count_;
- }
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public int getCountCount() {
- return count_.size();
- }
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public long getCount(int index) {
- return count_.get(index);
- }
- private int countMemoizedSerializedSize = -1;
-
- private void initFields() {
- count_ = java.util.Collections.emptyList();
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (getCountList().size() > 0) {
- output.writeRawVarint32(10);
- output.writeRawVarint32(countMemoizedSerializedSize);
- }
- for (int i = 0; i < count_.size(); i++) {
- output.writeUInt64NoTag(count_.get(i));
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- {
- int dataSize = 0;
- for (int i = 0; i < count_.size(); i++) {
- dataSize += com.google.protobuf.CodedOutputStream
- .computeUInt64SizeNoTag(count_.get(i));
- }
- size += dataSize;
- if (!getCountList().isEmpty()) {
- size += 1;
- size += com.google.protobuf.CodedOutputStream
- .computeInt32SizeNoTag(dataSize);
- }
- countMemoizedSerializedSize = dataSize;
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code orc.proto.BucketStatistics}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatisticsOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_BucketStatistics_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_BucketStatistics_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- count_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000001);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_orc_proto_BucketStatistics_descriptor;
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics getDefaultInstanceForType() {
- return org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics build() {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics buildPartial() {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics(this);
- int from_bitField0_ = bitField0_;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- count_ = java.util.Collections.unmodifiableList(count_);
- bitField0_ = (bitField0_ & ~0x00000001);
- }
- result.count_ = count_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics) {
- return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics other) {
- if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance()) return this;
- if (!other.count_.isEmpty()) {
- if (count_.isEmpty()) {
- count_ = other.count_;
- bitField0_ = (bitField0_ & ~0x00000001);
- } else {
- ensureCountIsMutable();
- count_.addAll(other.count_);
- }
- onChanged();
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // repeated uint64 count = 1 [packed = true];
- private java.util.List<java.lang.Long> count_ = java.util.Collections.emptyList();
- private void ensureCountIsMutable() {
- if (!((bitField0_ & 0x00000001) == 0x00000001)) {
- count_ = new java.util.ArrayList<java.lang.Long>(count_);
- bitField0_ |= 0x00000001;
- }
- }
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public java.util.List<java.lang.Long>
- getCountList() {
- return java.util.Collections.unmodifiableList(count_);
- }
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public int getCountCount() {
- return count_.size();
- }
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public long getCount(int index) {
- return count_.get(index);
- }
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public Builder setCount(
- int index, long value) {
- ensureCountIsMutable();
- count_.set(index, value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public Builder addCount(long value) {
- ensureCountIsMutable();
- count_.add(value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public Builder addAllCount(
- java.lang.Iterable<? extends java.lang.Long> values) {
- ensureCountIsMutable();
- super.addAll(values, count_);
- onChanged();
- return this;
- }
- /**
- * <code>repeated uint64 count = 1 [packed = true];</code>
- */
- public Builder clearCount() {
- count_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000001);
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:orc.proto.BucketStatistics)
- }
-
- static {
- defaultInstance = new BucketStatistics(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:orc.proto.BucketStatistics)
- }
-
- public interface DecimalStatisticsOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // optional string minimum = 1;
- /**
- * <code>optional string minimum = 1;</code>
- */
- boolean hasMinimum();
- /**
- * <code>optional string minimum = 1;</code>
- */
- java.lang.String getMinimum();
- /**
- * <code>optional string minimum = 1;</code>
- */
- com.google.protobuf.ByteString
- getMinimumBytes();
-
- // optional string maximum = 2;
- /**
- * <code>optional string maximum = 2;</code>
- */
- boolean hasMaximum();
- /**
- * <code>optional string maximum = 2;</code>
- */
- java.lang.String getMaximum();
- /**
- * <code>optional string maximum = 2;</code>
- */
- com.google.protobuf.ByteString
- getMaximumBytes();
-
- // optional string sum = 3;
- /**
- * <code>optional string sum = 3;</code>
- */
- boolean hasSum();
- /**
- * <code>optional string sum = 3;</code>
- */
- java.lang.String getSum();
- /**
- * <code>optional string sum = 3;</code>
- */
- com.google.protobuf.ByteString
- getSumBytes();
- }
- /**
- * Protobuf type {@code orc.proto.DecimalStatistics}
- */
- public static final class DecimalStatistics extends
- com.google.protobuf.GeneratedMessage
- implements DecimalStatisticsOrBuilder {
- // Use DecimalStatistics.newBuilder() to construct.
- private DecimalStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private DecimalStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final DecimalStatistics defaultInstance;
- public static DecimalStatistics getDefaultInstance() {
- return defaultInstance;
- }
-
- public DecimalStatistics getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private DecimalStatistics(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- bitField0_ |= 0x00000001;
- minimum_ = input.readBytes();
- break;
- }
- case 18: {
- bitField0_ |= 0x00000002;
- maximum_ = input.readBytes();
- break;
- }
- case 26: {
- bitField0_ |= 0x00000004;
- sum_ = input.readBytes();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.
<TRUNCATED>
[09/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestStringRedBlackTree.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestStringRedBlackTree.java b/orc/src/test/org/apache/orc/impl/TestStringRedBlackTree.java
new file mode 100644
index 0000000..3d4612c
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestStringRedBlackTree.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.orc.impl.RedBlackTree;
+import org.apache.orc.impl.StringRedBlackTree;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static junit.framework.Assert.assertEquals;
+
+/**
+ * Test the red-black tree with string keys.
+ */
+public class TestStringRedBlackTree {
+
+ /**
+ * Checks the red-black tree rules to make sure that we have correctly built
+ * a valid tree.
+ *
+ * Properties:
+ * 1. Red nodes must have black children
+ * 2. Each node must have the same black height on both sides.
+ *
+ * @param node The id of the root of the subtree to check for the red-black
+ * tree properties.
+ * @return The black-height of the subtree.
+ */
+ private int checkSubtree(RedBlackTree tree, int node, IntWritable count
+ ) throws IOException {
+ if (node == RedBlackTree.NULL) {
+ return 1;
+ }
+ count.set(count.get() + 1);
+ boolean is_red = tree.isRed(node);
+ int left = tree.getLeft(node);
+ int right = tree.getRight(node);
+ if (is_red) {
+ if (tree.isRed(left)) {
+ printTree(tree, "", tree.root);
+ throw new IllegalStateException("Left node of " + node + " is " + left +
+ " and both are red.");
+ }
+ if (tree.isRed(right)) {
+ printTree(tree, "", tree.root);
+ throw new IllegalStateException("Right node of " + node + " is " +
+ right + " and both are red.");
+ }
+ }
+ int left_depth = checkSubtree(tree, left, count);
+ int right_depth = checkSubtree(tree, right, count);
+ if (left_depth != right_depth) {
+ printTree(tree, "", tree.root);
+ throw new IllegalStateException("Lopsided tree at node " + node +
+ " with depths " + left_depth + " and " + right_depth);
+ }
+ if (is_red) {
+ return left_depth;
+ } else {
+ return left_depth + 1;
+ }
+ }
+
+ /**
+ * Checks the validity of the entire tree. Also ensures that the number of
+ * nodes visited is the same as the size of the set.
+ */
+ void checkTree(RedBlackTree tree) throws IOException {
+ IntWritable count = new IntWritable(0);
+ if (tree.isRed(tree.root)) {
+ printTree(tree, "", tree.root);
+ throw new IllegalStateException("root is red");
+ }
+ checkSubtree(tree, tree.root, count);
+ if (count.get() != tree.size) {
+ printTree(tree, "", tree.root);
+ throw new IllegalStateException("Broken tree! visited= " + count.get() +
+ " size=" + tree.size);
+ }
+ }
+
+ void printTree(RedBlackTree tree, String indent, int node
+ ) throws IOException {
+ if (node == RedBlackTree.NULL) {
+ System.err.println(indent + "NULL");
+ } else {
+ System.err.println(indent + "Node " + node + " color " +
+ (tree.isRed(node) ? "red" : "black"));
+ printTree(tree, indent + " ", tree.getLeft(node));
+ printTree(tree, indent + " ", tree.getRight(node));
+ }
+ }
+
+ private static class MyVisitor implements StringRedBlackTree.Visitor {
+ private final String[] words;
+ private final int[] order;
+ private final DataOutputBuffer buffer = new DataOutputBuffer();
+ int current = 0;
+
+ MyVisitor(String[] args, int[] order) {
+ words = args;
+ this.order = order;
+ }
+
+ @Override
+ public void visit(StringRedBlackTree.VisitorContext context
+ ) throws IOException {
+ String word = context.getText().toString();
+ assertEquals("in word " + current, words[current], word);
+ assertEquals("in word " + current, order[current],
+ context.getOriginalPosition());
+ buffer.reset();
+ context.writeBytes(buffer);
+ assertEquals(word, new String(buffer.getData(),0,buffer.getLength()));
+ current += 1;
+ }
+ }
+
+ void checkContents(StringRedBlackTree tree, int[] order,
+ String... params
+ ) throws IOException {
+ tree.visit(new MyVisitor(params, order));
+ }
+
+ StringRedBlackTree buildTree(String... params) throws IOException {
+ StringRedBlackTree result = new StringRedBlackTree(1000);
+ for(String word: params) {
+ result.add(word);
+ checkTree(result);
+ }
+ return result;
+ }
+
+ @Test
+ public void test1() throws Exception {
+ StringRedBlackTree tree = new StringRedBlackTree(5);
+ assertEquals(0, tree.getSizeInBytes());
+ checkTree(tree);
+ assertEquals(0, tree.add("owen"));
+ checkTree(tree);
+ assertEquals(1, tree.add("ashutosh"));
+ checkTree(tree);
+ assertEquals(0, tree.add("owen"));
+ checkTree(tree);
+ assertEquals(2, tree.add("alan"));
+ checkTree(tree);
+ assertEquals(2, tree.add("alan"));
+ checkTree(tree);
+ assertEquals(1, tree.add("ashutosh"));
+ checkTree(tree);
+ assertEquals(3, tree.add("greg"));
+ checkTree(tree);
+ assertEquals(4, tree.add("eric"));
+ checkTree(tree);
+ assertEquals(5, tree.add("arun"));
+ checkTree(tree);
+ assertEquals(6, tree.size());
+ checkTree(tree);
+ assertEquals(6, tree.add("eric14"));
+ checkTree(tree);
+ assertEquals(7, tree.add("o"));
+ checkTree(tree);
+ assertEquals(8, tree.add("ziggy"));
+ checkTree(tree);
+ assertEquals(9, tree.add("z"));
+ checkTree(tree);
+ checkContents(tree, new int[]{2,5,1,4,6,3,7,0,9,8},
+ "alan", "arun", "ashutosh", "eric", "eric14", "greg",
+ "o", "owen", "z", "ziggy");
+ assertEquals(32888, tree.getSizeInBytes());
+ // check that adding greg again bumps the count
+ assertEquals(3, tree.add("greg"));
+ assertEquals(41, tree.getCharacterSize());
+ // add some more strings to test the different branches of the
+ // rebalancing
+ assertEquals(10, tree.add("zak"));
+ checkTree(tree);
+ assertEquals(11, tree.add("eric1"));
+ checkTree(tree);
+ assertEquals(12, tree.add("ash"));
+ checkTree(tree);
+ assertEquals(13, tree.add("harry"));
+ checkTree(tree);
+ assertEquals(14, tree.add("john"));
+ checkTree(tree);
+ tree.clear();
+ checkTree(tree);
+ assertEquals(0, tree.getSizeInBytes());
+ assertEquals(0, tree.getCharacterSize());
+ }
+
+ @Test
+ public void test2() throws Exception {
+ StringRedBlackTree tree =
+ buildTree("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l",
+ "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
+ assertEquals(26, tree.size());
+ checkContents(tree, new int[]{0,1,2, 3,4,5, 6,7,8, 9,10,11, 12,13,14,
+ 15,16,17, 18,19,20, 21,22,23, 24,25},
+ "a", "b", "c", "d", "e", "f", "g", "h", "i", "j","k", "l", "m", "n", "o",
+ "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
+ }
+
+ @Test
+ public void test3() throws Exception {
+ StringRedBlackTree tree =
+ buildTree("z", "y", "x", "w", "v", "u", "t", "s", "r", "q", "p", "o", "n",
+ "m", "l", "k", "j", "i", "h", "g", "f", "e", "d", "c", "b", "a");
+ assertEquals(26, tree.size());
+ checkContents(tree, new int[]{25,24,23, 22,21,20, 19,18,17, 16,15,14,
+ 13,12,11, 10,9,8, 7,6,5, 4,3,2, 1,0},
+ "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
+ "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestZlib.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestZlib.java b/orc/src/test/org/apache/orc/impl/TestZlib.java
new file mode 100644
index 0000000..327ecfc
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestZlib.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.orc.CompressionCodec;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.fail;
+
+public class TestZlib {
+
+ @Test
+ public void testNoOverflow() throws Exception {
+ ByteBuffer in = ByteBuffer.allocate(10);
+ ByteBuffer out = ByteBuffer.allocate(10);
+ in.put(new byte[]{1,2,3,4,5,6,7,10});
+ in.flip();
+ CompressionCodec codec = new ZlibCodec();
+ assertEquals(false, codec.compress(in, out, null));
+ }
+
+ @Test
+ public void testCorrupt() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(1000);
+ buf.put(new byte[]{127,-128,0,99,98,-1});
+ buf.flip();
+ CompressionCodec codec = new ZlibCodec();
+ ByteBuffer out = ByteBuffer.allocate(1000);
+ try {
+ codec.decompress(buf, out);
+ fail();
+ } catch (IOException ioe) {
+ // EXPECTED
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a2ea91e..fb62e2d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -44,6 +44,7 @@
<module>jdbc</module>
<module>metastore</module>
<module>odbc</module>
+ <module>orc</module>
<module>ql</module>
<module>serde</module>
<module>service</module>
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/pom.xml
----------------------------------------------------------------------
diff --git a/ql/pom.xml b/ql/pom.xml
index ee1d46c..cf6fad3 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -83,6 +83,11 @@
<version>${kryo.version}</version>
</dependency>
<dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop-bundle</artifactId>
<version>${parquet.version}</version>
@@ -118,6 +123,11 @@
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
+ <groupId>org.iq80.snappy</groupId>
+ <artifactId>snappy</artifactId>
+ <version>${snappy.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>${commons-lang.version}</version>
@@ -395,11 +405,6 @@
<version>${gson.version}</version>
</dependency>
<dependency>
- <groupId>org.iq80.snappy</groupId>
- <artifactId>snappy</artifactId>
- <version>${snappy.version}</version>
- </dependency>
- <dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>${json.version}</version>
@@ -622,42 +627,6 @@
</dependency>
</dependencies>
- <profiles>
- <profile>
- <id>protobuf</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>generate-protobuf-sources</id>
- <phase>generate-sources</phase>
- <configuration>
- <target>
- <property name="protobuf.src.dir" location="${basedir}/src/protobuf"/>
- <property name="protobuf.build.dir" location="${basedir}/src/gen/protobuf/gen-java"/>
- <echo>Building ORC Protobuf</echo>
- <mkdir dir="${protobuf.build.dir}"/>
- <exec executable="protoc" failonerror="true">
- <arg value="--java_out=${protobuf.build.dir}"/>
- <arg value="-I=${protobuf.src.dir}/org/apache/hadoop/hive/ql/io/orc"/>
- <arg value="${protobuf.src.dir}/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto"/>
- </exec>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
<build>
<sourceDirectory>${basedir}/src/java</sourceDirectory>
<testSourceDirectory>${basedir}/src/test</testSourceDirectory>
@@ -772,7 +741,6 @@
<include>org.apache.hive.shims:hive-shims-0.23</include>
<include>org.apache.hive.shims:hive-shims-0.23</include>
<include>org.apache.hive.shims:hive-shims-common</include>
- <include>org.apache.hive:hive-storage-api</include>
<include>com.googlecode.javaewah:JavaEWAH</include>
<include>javolution:javolution</include>
<include>com.google.protobuf:protobuf-java</include>
@@ -783,6 +751,7 @@
<include>net.sf.opencsv:opencsv</include>
<include>org.apache.hive:spark-client</include>
<include>org.apache.hive:hive-storage-api</include>
+ <include>org.apache.hive:hive-orc</include>
<include>joda-time:joda-time</include>
</includes>
</artifactSet>
@@ -812,7 +781,6 @@
</goals>
<configuration>
<sources>
- <source>src/gen/protobuf/gen-java</source>
<source>src/gen/thrift/gen-javabean</source>
<source>${project.build.directory}/generated-sources/java</source>
</sources>
[07/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
index 99a3e8d..2c9deac 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
@@ -20,16 +20,16 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.IOException;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.hadoop.hive.ql.io.orc.Writer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
+import org.apache.orc.CompressionKind;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.OrcFileKeyWrapper;
import org.apache.hadoop.hive.ql.io.orc.OrcFileValueWrapper;
import org.apache.hadoop.hive.ql.io.orc.Reader;
-import org.apache.hadoop.hive.ql.io.orc.Writer;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.OrcFileMergeDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/filters/BloomFilterIO.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/filters/BloomFilterIO.java b/ql/src/java/org/apache/hadoop/hive/ql/io/filters/BloomFilterIO.java
index 56aec9f..878efbe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/filters/BloomFilterIO.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/filters/BloomFilterIO.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.ql.io.filters;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto;
+import org.apache.orc.OrcProto;
import org.apache.hive.common.util.BloomFilter;
import com.google.common.primitives.Longs;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BinaryColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BinaryColumnStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BinaryColumnStatistics.java
deleted file mode 100644
index 23030a3..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BinaryColumnStatistics.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * Statistics for binary columns.
- */
-public interface BinaryColumnStatistics extends ColumnStatistics {
- long getSum();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java
deleted file mode 100644
index ec1f0a9..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-
-public class BitFieldReader {
- private final RunLengthByteReader input;
- /** The number of bits in one item. Non-test code always uses 1. */
- private final int bitSize;
- private int current;
- private int bitsLeft;
- private final int mask;
-
- public BitFieldReader(InStream input,
- int bitSize) throws IOException {
- this.input = new RunLengthByteReader(input);
- this.bitSize = bitSize;
- mask = (1 << bitSize) - 1;
- }
-
- public void setInStream(InStream inStream) {
- this.input.setInStream(inStream);
- }
-
- private void readByte() throws IOException {
- if (input.hasNext()) {
- current = 0xff & input.next();
- bitsLeft = 8;
- } else {
- throw new EOFException("Read past end of bit field from " + this);
- }
- }
-
- public int next() throws IOException {
- int result = 0;
- int bitsLeftToRead = bitSize;
- while (bitsLeftToRead > bitsLeft) {
- result <<= bitsLeft;
- result |= current & ((1 << bitsLeft) - 1);
- bitsLeftToRead -= bitsLeft;
- readByte();
- }
- if (bitsLeftToRead > 0) {
- result <<= bitsLeftToRead;
- bitsLeft -= bitsLeftToRead;
- result |= (current >>> bitsLeft) & ((1 << bitsLeftToRead) - 1);
- }
- return result & mask;
- }
-
- /**
- * Unlike integer readers, where runs are encoded explicitly, in this one we have to read ahead
- * to figure out whether we have a run. Given that runs in booleans are likely it's worth it.
- * However it means we'd need to keep track of how many bytes we read, and next/nextVector won't
- * work anymore once this is called. These is trivial to fix, but these are never interspersed.
- */
- private boolean lastRunValue;
- private int lastRunLength = -1;
- private void readNextRun(int maxRunLength) throws IOException {
- assert bitSize == 1;
- if (lastRunLength > 0) return; // last run is not exhausted yet
- if (bitsLeft == 0) {
- readByte();
- }
- // First take care of the partial bits.
- boolean hasVal = false;
- int runLength = 0;
- if (bitsLeft != 8) {
- int partialBitsMask = (1 << bitsLeft) - 1;
- int partialBits = current & partialBitsMask;
- if (partialBits == partialBitsMask || partialBits == 0) {
- lastRunValue = (partialBits == partialBitsMask);
- if (maxRunLength <= bitsLeft) {
- lastRunLength = maxRunLength;
- return;
- }
- maxRunLength -= bitsLeft;
- hasVal = true;
- runLength = bitsLeft;
- bitsLeft = 0;
- } else {
- // There's no run in partial bits. Return whatever we have.
- int prefixBitsCount = 32 - bitsLeft;
- runLength = Integer.numberOfLeadingZeros(partialBits) - prefixBitsCount;
- lastRunValue = (runLength > 0);
- lastRunLength = Math.min(maxRunLength, lastRunValue ? runLength :
- (Integer.numberOfLeadingZeros(~(partialBits | ~partialBitsMask)) - prefixBitsCount));
- return;
- }
- assert bitsLeft == 0;
- readByte();
- }
- if (!hasVal) {
- lastRunValue = ((current >> 7) == 1);
- hasVal = true;
- }
- // Read full bytes until the run ends.
- assert bitsLeft == 8;
- while (maxRunLength >= 8
- && ((lastRunValue && (current == 0xff)) || (!lastRunValue && (current == 0)))) {
- runLength += 8;
- maxRunLength -= 8;
- readByte();
- }
- if (maxRunLength > 0) {
- int extraBits = Integer.numberOfLeadingZeros(
- lastRunValue ? (~(current | ~255)) : current) - 24;
- bitsLeft -= extraBits;
- runLength += extraBits;
- }
- lastRunLength = runLength;
- }
-
- void nextVector(LongColumnVector previous, long previousLen) throws IOException {
- previous.isRepeating = true;
- for (int i = 0; i < previousLen; i++) {
- if (!previous.isNull[i]) {
- previous.vector[i] = next();
- } else {
- // The default value of null for int types in vectorized
- // processing is 1, so set that if the value is null
- previous.vector[i] = 1;
- }
-
- // The default value for nulls in Vectorization for int types is 1
- // and given that non null value can also be 1, we need to check for isNull also
- // when determining the isRepeating flag.
- if (previous.isRepeating
- && i > 0
- && ((previous.vector[i - 1] != previous.vector[i]) || (previous.isNull[i - 1] != previous.isNull[i]))) {
- previous.isRepeating = false;
- }
- }
- }
-
- public void seek(PositionProvider index) throws IOException {
- input.seek(index);
- int consumed = (int) index.getNext();
- if (consumed > 8) {
- throw new IllegalArgumentException("Seek past end of byte at " +
- consumed + " in " + input);
- } else if (consumed != 0) {
- readByte();
- bitsLeft = 8 - consumed;
- } else {
- bitsLeft = 0;
- }
- }
-
- void skip(long items) throws IOException {
- long totalBits = bitSize * items;
- if (bitsLeft >= totalBits) {
- bitsLeft -= totalBits;
- } else {
- totalBits -= bitsLeft;
- input.skip(totalBits / 8);
- current = input.next();
- bitsLeft = (int) (8 - (totalBits % 8));
- }
- }
-
- @Override
- public String toString() {
- return "bit reader current: " + current + " bits left: " + bitsLeft +
- " bit size: " + bitSize + " from " + input;
- }
-
- boolean hasFullByte() {
- return bitsLeft == 8 || bitsLeft == 0;
- }
-
- int peekOneBit() throws IOException {
- assert bitSize == 1;
- if (bitsLeft == 0) {
- readByte();
- }
- return (current >>> (bitsLeft - 1)) & 1;
- }
-
- int peekFullByte() throws IOException {
- assert bitSize == 1;
- assert bitsLeft == 8 || bitsLeft == 0;
- if (bitsLeft == 0) {
- readByte();
- }
- return current;
- }
-
- void skipInCurrentByte(int bits) throws IOException {
- assert bitsLeft >= bits;
- bitsLeft -= bits;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldWriter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldWriter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldWriter.java
deleted file mode 100644
index 0608da6..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldWriter.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-
-class BitFieldWriter {
- private RunLengthByteWriter output;
- private final int bitSize;
- private byte current = 0;
- private int bitsLeft = 8;
-
- BitFieldWriter(PositionedOutputStream output,
- int bitSize) throws IOException {
- this.output = new RunLengthByteWriter(output);
- this.bitSize = bitSize;
- }
-
- private void writeByte() throws IOException {
- output.write(current);
- current = 0;
- bitsLeft = 8;
- }
-
- void flush() throws IOException {
- if (bitsLeft != 8) {
- writeByte();
- }
- output.flush();
- }
-
- void write(int value) throws IOException {
- int bitsToWrite = bitSize;
- while (bitsToWrite > bitsLeft) {
- // add the bits to the bottom of the current word
- current |= value >>> (bitsToWrite - bitsLeft);
- // subtract out the bits we just added
- bitsToWrite -= bitsLeft;
- // zero out the bits above bitsToWrite
- value &= (1 << bitsToWrite) - 1;
- writeByte();
- }
- bitsLeft -= bitsToWrite;
- current |= value << bitsLeft;
- if (bitsLeft == 0) {
- writeByte();
- }
- }
-
- void getPosition(PositionRecorder recorder) throws IOException {
- output.getPosition(recorder);
- recorder.addPosition(8 - bitsLeft);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BooleanColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BooleanColumnStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BooleanColumnStatistics.java
deleted file mode 100644
index 6d03998..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BooleanColumnStatistics.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * Statistics for boolean columns.
- */
-public interface BooleanColumnStatistics extends ColumnStatistics {
- long getFalseCount();
-
- long getTrueCount();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatistics.java
deleted file mode 100644
index 7c973c2..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatistics.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * Statistics that are available for all types of columns.
- */
-public interface ColumnStatistics {
- /**
- * Get the number of values in this column. It will differ from the number
- * of rows because of NULL values and repeated values.
- * @return the number of values
- */
- long getNumberOfValues();
-
- /**
- * Returns true if there are nulls in the scope of column statistics.
- * @return true if null present else false
- */
- boolean hasNull();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
deleted file mode 100644
index bcca9de..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
+++ /dev/null
@@ -1,1082 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.sql.Date;
-import java.sql.Timestamp;
-
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparator;
-
-class ColumnStatisticsImpl implements ColumnStatistics {
-
- private static final class BooleanStatisticsImpl extends ColumnStatisticsImpl
- implements BooleanColumnStatistics {
- private long trueCount = 0;
-
- BooleanStatisticsImpl(OrcProto.ColumnStatistics stats) {
- super(stats);
- OrcProto.BucketStatistics bkt = stats.getBucketStatistics();
- trueCount = bkt.getCount(0);
- }
-
- BooleanStatisticsImpl() {
- }
-
- @Override
- void reset() {
- super.reset();
- trueCount = 0;
- }
-
- @Override
- void updateBoolean(boolean value, int repetitions) {
- if (value) {
- trueCount += repetitions;
- }
- }
-
- @Override
- void merge(ColumnStatisticsImpl other) {
- if (other instanceof BooleanStatisticsImpl) {
- BooleanStatisticsImpl bkt = (BooleanStatisticsImpl) other;
- trueCount += bkt.trueCount;
- } else {
- if (isStatsExists() && trueCount != 0) {
- throw new IllegalArgumentException("Incompatible merging of boolean column statistics");
- }
- }
- super.merge(other);
- }
-
- @Override
- OrcProto.ColumnStatistics.Builder serialize() {
- OrcProto.ColumnStatistics.Builder builder = super.serialize();
- OrcProto.BucketStatistics.Builder bucket =
- OrcProto.BucketStatistics.newBuilder();
- bucket.addCount(trueCount);
- builder.setBucketStatistics(bucket);
- return builder;
- }
-
- @Override
- public long getFalseCount() {
- return getNumberOfValues() - trueCount;
- }
-
- @Override
- public long getTrueCount() {
- return trueCount;
- }
-
- @Override
- public String toString() {
- return super.toString() + " true: " + trueCount;
- }
- }
-
- private static final class IntegerStatisticsImpl extends ColumnStatisticsImpl
- implements IntegerColumnStatistics {
-
- private long minimum = Long.MAX_VALUE;
- private long maximum = Long.MIN_VALUE;
- private long sum = 0;
- private boolean hasMinimum = false;
- private boolean overflow = false;
-
- IntegerStatisticsImpl() {
- }
-
- IntegerStatisticsImpl(OrcProto.ColumnStatistics stats) {
- super(stats);
- OrcProto.IntegerStatistics intStat = stats.getIntStatistics();
- if (intStat.hasMinimum()) {
- hasMinimum = true;
- minimum = intStat.getMinimum();
- }
- if (intStat.hasMaximum()) {
- maximum = intStat.getMaximum();
- }
- if (intStat.hasSum()) {
- sum = intStat.getSum();
- } else {
- overflow = true;
- }
- }
-
- @Override
- void reset() {
- super.reset();
- hasMinimum = false;
- minimum = Long.MAX_VALUE;
- maximum = Long.MIN_VALUE;
- sum = 0;
- overflow = false;
- }
-
- @Override
- void updateInteger(long value, int repetitions) {
- if (!hasMinimum) {
- hasMinimum = true;
- minimum = value;
- maximum = value;
- } else if (value < minimum) {
- minimum = value;
- } else if (value > maximum) {
- maximum = value;
- }
- if (!overflow) {
- boolean wasPositive = sum >= 0;
- sum += value * repetitions;
- if ((value >= 0) == wasPositive) {
- overflow = (sum >= 0) != wasPositive;
- }
- }
- }
-
- @Override
- void merge(ColumnStatisticsImpl other) {
- if (other instanceof IntegerStatisticsImpl) {
- IntegerStatisticsImpl otherInt = (IntegerStatisticsImpl) other;
- if (!hasMinimum) {
- hasMinimum = otherInt.hasMinimum;
- minimum = otherInt.minimum;
- maximum = otherInt.maximum;
- } else if (otherInt.hasMinimum) {
- if (otherInt.minimum < minimum) {
- minimum = otherInt.minimum;
- }
- if (otherInt.maximum > maximum) {
- maximum = otherInt.maximum;
- }
- }
-
- overflow |= otherInt.overflow;
- if (!overflow) {
- boolean wasPositive = sum >= 0;
- sum += otherInt.sum;
- if ((otherInt.sum >= 0) == wasPositive) {
- overflow = (sum >= 0) != wasPositive;
- }
- }
- } else {
- if (isStatsExists() && hasMinimum) {
- throw new IllegalArgumentException("Incompatible merging of integer column statistics");
- }
- }
- super.merge(other);
- }
-
- @Override
- OrcProto.ColumnStatistics.Builder serialize() {
- OrcProto.ColumnStatistics.Builder builder = super.serialize();
- OrcProto.IntegerStatistics.Builder intb =
- OrcProto.IntegerStatistics.newBuilder();
- if (hasMinimum) {
- intb.setMinimum(minimum);
- intb.setMaximum(maximum);
- }
- if (!overflow) {
- intb.setSum(sum);
- }
- builder.setIntStatistics(intb);
- return builder;
- }
-
- @Override
- public long getMinimum() {
- return minimum;
- }
-
- @Override
- public long getMaximum() {
- return maximum;
- }
-
- @Override
- public boolean isSumDefined() {
- return !overflow;
- }
-
- @Override
- public long getSum() {
- return sum;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder(super.toString());
- if (hasMinimum) {
- buf.append(" min: ");
- buf.append(minimum);
- buf.append(" max: ");
- buf.append(maximum);
- }
- if (!overflow) {
- buf.append(" sum: ");
- buf.append(sum);
- }
- return buf.toString();
- }
- }
-
- private static final class DoubleStatisticsImpl extends ColumnStatisticsImpl
- implements DoubleColumnStatistics {
- private boolean hasMinimum = false;
- private double minimum = Double.MAX_VALUE;
- private double maximum = Double.MIN_VALUE;
- private double sum = 0;
-
- DoubleStatisticsImpl() {
- }
-
- DoubleStatisticsImpl(OrcProto.ColumnStatistics stats) {
- super(stats);
- OrcProto.DoubleStatistics dbl = stats.getDoubleStatistics();
- if (dbl.hasMinimum()) {
- hasMinimum = true;
- minimum = dbl.getMinimum();
- }
- if (dbl.hasMaximum()) {
- maximum = dbl.getMaximum();
- }
- if (dbl.hasSum()) {
- sum = dbl.getSum();
- }
- }
-
- @Override
- void reset() {
- super.reset();
- hasMinimum = false;
- minimum = Double.MAX_VALUE;
- maximum = Double.MIN_VALUE;
- sum = 0;
- }
-
- @Override
- void updateDouble(double value) {
- if (!hasMinimum) {
- hasMinimum = true;
- minimum = value;
- maximum = value;
- } else if (value < minimum) {
- minimum = value;
- } else if (value > maximum) {
- maximum = value;
- }
- sum += value;
- }
-
- @Override
- void merge(ColumnStatisticsImpl other) {
- if (other instanceof DoubleStatisticsImpl) {
- DoubleStatisticsImpl dbl = (DoubleStatisticsImpl) other;
- if (!hasMinimum) {
- hasMinimum = dbl.hasMinimum;
- minimum = dbl.minimum;
- maximum = dbl.maximum;
- } else if (dbl.hasMinimum) {
- if (dbl.minimum < minimum) {
- minimum = dbl.minimum;
- }
- if (dbl.maximum > maximum) {
- maximum = dbl.maximum;
- }
- }
- sum += dbl.sum;
- } else {
- if (isStatsExists() && hasMinimum) {
- throw new IllegalArgumentException("Incompatible merging of double column statistics");
- }
- }
- super.merge(other);
- }
-
- @Override
- OrcProto.ColumnStatistics.Builder serialize() {
- OrcProto.ColumnStatistics.Builder builder = super.serialize();
- OrcProto.DoubleStatistics.Builder dbl =
- OrcProto.DoubleStatistics.newBuilder();
- if (hasMinimum) {
- dbl.setMinimum(minimum);
- dbl.setMaximum(maximum);
- }
- dbl.setSum(sum);
- builder.setDoubleStatistics(dbl);
- return builder;
- }
-
- @Override
- public double getMinimum() {
- return minimum;
- }
-
- @Override
- public double getMaximum() {
- return maximum;
- }
-
- @Override
- public double getSum() {
- return sum;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder(super.toString());
- if (hasMinimum) {
- buf.append(" min: ");
- buf.append(minimum);
- buf.append(" max: ");
- buf.append(maximum);
- }
- buf.append(" sum: ");
- buf.append(sum);
- return buf.toString();
- }
- }
-
- protected static final class StringStatisticsImpl extends ColumnStatisticsImpl
- implements StringColumnStatistics {
- private Text minimum = null;
- private Text maximum = null;
- private long sum = 0;
-
- StringStatisticsImpl() {
- }
-
- StringStatisticsImpl(OrcProto.ColumnStatistics stats) {
- super(stats);
- OrcProto.StringStatistics str = stats.getStringStatistics();
- if (str.hasMaximum()) {
- maximum = new Text(str.getMaximum());
- }
- if (str.hasMinimum()) {
- minimum = new Text(str.getMinimum());
- }
- if(str.hasSum()) {
- sum = str.getSum();
- }
- }
-
- @Override
- void reset() {
- super.reset();
- minimum = null;
- maximum = null;
- sum = 0;
- }
-
- @Override
- void updateString(Text value) {
- if (minimum == null) {
- maximum = minimum = new Text(value);
- } else if (minimum.compareTo(value) > 0) {
- minimum = new Text(value);
- } else if (maximum.compareTo(value) < 0) {
- maximum = new Text(value);
- }
- sum += value.getLength();
- }
-
- @Override
- void updateString(byte[] bytes, int offset, int length, int repetitions) {
- if (minimum == null) {
- maximum = minimum = new Text();
- maximum.set(bytes, offset, length);
- } else if (WritableComparator.compareBytes(minimum.getBytes(), 0,
- minimum.getLength(), bytes, offset, length) > 0) {
- minimum = new Text();
- minimum.set(bytes, offset, length);
- } else if (WritableComparator.compareBytes(maximum.getBytes(), 0,
- maximum.getLength(), bytes, offset, length) < 0) {
- maximum = new Text();
- maximum.set(bytes, offset, length);
- }
- sum += length * repetitions;
- }
-
- @Override
- void merge(ColumnStatisticsImpl other) {
- if (other instanceof StringStatisticsImpl) {
- StringStatisticsImpl str = (StringStatisticsImpl) other;
- if (minimum == null) {
- if (str.minimum != null) {
- maximum = new Text(str.getMaximum());
- minimum = new Text(str.getMinimum());
- } else {
- /* both are empty */
- maximum = minimum = null;
- }
- } else if (str.minimum != null) {
- if (minimum.compareTo(str.minimum) > 0) {
- minimum = new Text(str.getMinimum());
- }
- if (maximum.compareTo(str.maximum) < 0) {
- maximum = new Text(str.getMaximum());
- }
- }
- sum += str.sum;
- } else {
- if (isStatsExists() && minimum != null) {
- throw new IllegalArgumentException("Incompatible merging of string column statistics");
- }
- }
- super.merge(other);
- }
-
- @Override
- OrcProto.ColumnStatistics.Builder serialize() {
- OrcProto.ColumnStatistics.Builder result = super.serialize();
- OrcProto.StringStatistics.Builder str =
- OrcProto.StringStatistics.newBuilder();
- if (getNumberOfValues() != 0) {
- str.setMinimum(getMinimum());
- str.setMaximum(getMaximum());
- str.setSum(sum);
- }
- result.setStringStatistics(str);
- return result;
- }
-
- @Override
- public String getMinimum() {
- return minimum == null ? null : minimum.toString();
- }
-
- @Override
- public String getMaximum() {
- return maximum == null ? null : maximum.toString();
- }
-
- @Override
- public long getSum() {
- return sum;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder(super.toString());
- if (getNumberOfValues() != 0) {
- buf.append(" min: ");
- buf.append(getMinimum());
- buf.append(" max: ");
- buf.append(getMaximum());
- buf.append(" sum: ");
- buf.append(sum);
- }
- return buf.toString();
- }
- }
-
- protected static final class BinaryStatisticsImpl extends ColumnStatisticsImpl implements
- BinaryColumnStatistics {
-
- private long sum = 0;
-
- BinaryStatisticsImpl() {
- }
-
- BinaryStatisticsImpl(OrcProto.ColumnStatistics stats) {
- super(stats);
- OrcProto.BinaryStatistics binStats = stats.getBinaryStatistics();
- if (binStats.hasSum()) {
- sum = binStats.getSum();
- }
- }
-
- @Override
- void reset() {
- super.reset();
- sum = 0;
- }
-
- @Override
- void updateBinary(BytesWritable value) {
- sum += value.getLength();
- }
-
- @Override
- void updateBinary(byte[] bytes, int offset, int length, int repetitions) {
- sum += length * repetitions;
- }
-
- @Override
- void merge(ColumnStatisticsImpl other) {
- if (other instanceof BinaryColumnStatistics) {
- BinaryStatisticsImpl bin = (BinaryStatisticsImpl) other;
- sum += bin.sum;
- } else {
- if (isStatsExists() && sum != 0) {
- throw new IllegalArgumentException("Incompatible merging of binary column statistics");
- }
- }
- super.merge(other);
- }
-
- @Override
- public long getSum() {
- return sum;
- }
-
- @Override
- OrcProto.ColumnStatistics.Builder serialize() {
- OrcProto.ColumnStatistics.Builder result = super.serialize();
- OrcProto.BinaryStatistics.Builder bin = OrcProto.BinaryStatistics.newBuilder();
- bin.setSum(sum);
- result.setBinaryStatistics(bin);
- return result;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder(super.toString());
- if (getNumberOfValues() != 0) {
- buf.append(" sum: ");
- buf.append(sum);
- }
- return buf.toString();
- }
- }
-
- private static final class DecimalStatisticsImpl extends ColumnStatisticsImpl
- implements DecimalColumnStatistics {
- private HiveDecimal minimum = null;
- private HiveDecimal maximum = null;
- private HiveDecimal sum = HiveDecimal.ZERO;
-
- DecimalStatisticsImpl() {
- }
-
- DecimalStatisticsImpl(OrcProto.ColumnStatistics stats) {
- super(stats);
- OrcProto.DecimalStatistics dec = stats.getDecimalStatistics();
- if (dec.hasMaximum()) {
- maximum = HiveDecimal.create(dec.getMaximum());
- }
- if (dec.hasMinimum()) {
- minimum = HiveDecimal.create(dec.getMinimum());
- }
- if (dec.hasSum()) {
- sum = HiveDecimal.create(dec.getSum());
- } else {
- sum = null;
- }
- }
-
- @Override
- void reset() {
- super.reset();
- minimum = null;
- maximum = null;
- sum = HiveDecimal.ZERO;
- }
-
- @Override
- void updateDecimal(HiveDecimal value) {
- if (minimum == null) {
- minimum = value;
- maximum = value;
- } else if (minimum.compareTo(value) > 0) {
- minimum = value;
- } else if (maximum.compareTo(value) < 0) {
- maximum = value;
- }
- if (sum != null) {
- sum = sum.add(value);
- }
- }
-
- @Override
- void merge(ColumnStatisticsImpl other) {
- if (other instanceof DecimalStatisticsImpl) {
- DecimalStatisticsImpl dec = (DecimalStatisticsImpl) other;
- if (minimum == null) {
- minimum = dec.minimum;
- maximum = dec.maximum;
- sum = dec.sum;
- } else if (dec.minimum != null) {
- if (minimum.compareTo(dec.minimum) > 0) {
- minimum = dec.minimum;
- }
- if (maximum.compareTo(dec.maximum) < 0) {
- maximum = dec.maximum;
- }
- if (sum == null || dec.sum == null) {
- sum = null;
- } else {
- sum = sum.add(dec.sum);
- }
- }
- } else {
- if (isStatsExists() && minimum != null) {
- throw new IllegalArgumentException("Incompatible merging of decimal column statistics");
- }
- }
- super.merge(other);
- }
-
- @Override
- OrcProto.ColumnStatistics.Builder serialize() {
- OrcProto.ColumnStatistics.Builder result = super.serialize();
- OrcProto.DecimalStatistics.Builder dec =
- OrcProto.DecimalStatistics.newBuilder();
- if (getNumberOfValues() != 0 && minimum != null) {
- dec.setMinimum(minimum.toString());
- dec.setMaximum(maximum.toString());
- }
- if (sum != null) {
- dec.setSum(sum.toString());
- }
- result.setDecimalStatistics(dec);
- return result;
- }
-
- @Override
- public HiveDecimal getMinimum() {
- return minimum;
- }
-
- @Override
- public HiveDecimal getMaximum() {
- return maximum;
- }
-
- @Override
- public HiveDecimal getSum() {
- return sum;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder(super.toString());
- if (getNumberOfValues() != 0) {
- buf.append(" min: ");
- buf.append(minimum);
- buf.append(" max: ");
- buf.append(maximum);
- if (sum != null) {
- buf.append(" sum: ");
- buf.append(sum);
- }
- }
- return buf.toString();
- }
- }
-
- private static final class DateStatisticsImpl extends ColumnStatisticsImpl
- implements DateColumnStatistics {
- private Integer minimum = null;
- private Integer maximum = null;
-
- DateStatisticsImpl() {
- }
-
- DateStatisticsImpl(OrcProto.ColumnStatistics stats) {
- super(stats);
- OrcProto.DateStatistics dateStats = stats.getDateStatistics();
- // min,max values serialized/deserialized as int (days since epoch)
- if (dateStats.hasMaximum()) {
- maximum = dateStats.getMaximum();
- }
- if (dateStats.hasMinimum()) {
- minimum = dateStats.getMinimum();
- }
- }
-
- @Override
- void reset() {
- super.reset();
- minimum = null;
- maximum = null;
- }
-
- @Override
- void updateDate(DateWritable value) {
- if (minimum == null) {
- minimum = value.getDays();
- maximum = value.getDays();
- } else if (minimum > value.getDays()) {
- minimum = value.getDays();
- } else if (maximum < value.getDays()) {
- maximum = value.getDays();
- }
- }
-
- @Override
- void updateDate(int value) {
- if (minimum == null) {
- minimum = value;
- maximum = value;
- } else if (minimum > value) {
- minimum = value;
- } else if (maximum < value) {
- maximum = value;
- }
- }
-
- @Override
- void merge(ColumnStatisticsImpl other) {
- if (other instanceof DateStatisticsImpl) {
- DateStatisticsImpl dateStats = (DateStatisticsImpl) other;
- if (minimum == null) {
- minimum = dateStats.minimum;
- maximum = dateStats.maximum;
- } else if (dateStats.minimum != null) {
- if (minimum > dateStats.minimum) {
- minimum = dateStats.minimum;
- }
- if (maximum < dateStats.maximum) {
- maximum = dateStats.maximum;
- }
- }
- } else {
- if (isStatsExists() && minimum != null) {
- throw new IllegalArgumentException("Incompatible merging of date column statistics");
- }
- }
- super.merge(other);
- }
-
- @Override
- OrcProto.ColumnStatistics.Builder serialize() {
- OrcProto.ColumnStatistics.Builder result = super.serialize();
- OrcProto.DateStatistics.Builder dateStats =
- OrcProto.DateStatistics.newBuilder();
- if (getNumberOfValues() != 0 && minimum != null) {
- dateStats.setMinimum(minimum);
- dateStats.setMaximum(maximum);
- }
- result.setDateStatistics(dateStats);
- return result;
- }
-
- private transient final DateWritable minDate = new DateWritable();
- private transient final DateWritable maxDate = new DateWritable();
-
- @Override
- public Date getMinimum() {
- if (minimum == null) {
- return null;
- }
- minDate.set(minimum);
- return minDate.get();
- }
-
- @Override
- public Date getMaximum() {
- if (maximum == null) {
- return null;
- }
- maxDate.set(maximum);
- return maxDate.get();
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder(super.toString());
- if (getNumberOfValues() != 0) {
- buf.append(" min: ");
- buf.append(getMinimum());
- buf.append(" max: ");
- buf.append(getMaximum());
- }
- return buf.toString();
- }
- }
-
- private static final class TimestampStatisticsImpl extends ColumnStatisticsImpl
- implements TimestampColumnStatistics {
- private Long minimum = null;
- private Long maximum = null;
-
- TimestampStatisticsImpl() {
- }
-
- TimestampStatisticsImpl(OrcProto.ColumnStatistics stats) {
- super(stats);
- OrcProto.TimestampStatistics timestampStats = stats.getTimestampStatistics();
- // min,max values serialized/deserialized as int (milliseconds since epoch)
- if (timestampStats.hasMaximum()) {
- maximum = timestampStats.getMaximum();
- }
- if (timestampStats.hasMinimum()) {
- minimum = timestampStats.getMinimum();
- }
- }
-
- @Override
- void reset() {
- super.reset();
- minimum = null;
- maximum = null;
- }
-
- @Override
- void updateTimestamp(Timestamp value) {
- if (minimum == null) {
- minimum = value.getTime();
- maximum = value.getTime();
- } else if (minimum > value.getTime()) {
- minimum = value.getTime();
- } else if (maximum < value.getTime()) {
- maximum = value.getTime();
- }
- }
-
- @Override
- void updateTimestamp(long value) {
- if (minimum == null) {
- minimum = value;
- maximum = value;
- } else if (minimum > value) {
- minimum = value;
- } else if (maximum < value) {
- maximum = value;
- }
- }
-
- @Override
- void merge(ColumnStatisticsImpl other) {
- if (other instanceof TimestampStatisticsImpl) {
- TimestampStatisticsImpl timestampStats = (TimestampStatisticsImpl) other;
- if (minimum == null) {
- minimum = timestampStats.minimum;
- maximum = timestampStats.maximum;
- } else if (timestampStats.minimum != null) {
- if (minimum > timestampStats.minimum) {
- minimum = timestampStats.minimum;
- }
- if (maximum < timestampStats.maximum) {
- maximum = timestampStats.maximum;
- }
- }
- } else {
- if (isStatsExists() && minimum != null) {
- throw new IllegalArgumentException("Incompatible merging of timestamp column statistics");
- }
- }
- super.merge(other);
- }
-
- @Override
- OrcProto.ColumnStatistics.Builder serialize() {
- OrcProto.ColumnStatistics.Builder result = super.serialize();
- OrcProto.TimestampStatistics.Builder timestampStats = OrcProto.TimestampStatistics
- .newBuilder();
- if (getNumberOfValues() != 0 && minimum != null) {
- timestampStats.setMinimum(minimum);
- timestampStats.setMaximum(maximum);
- }
- result.setTimestampStatistics(timestampStats);
- return result;
- }
-
- @Override
- public Timestamp getMinimum() {
- return minimum == null ? null : new Timestamp(minimum);
- }
-
- @Override
- public Timestamp getMaximum() {
- return maximum == null ? null : new Timestamp(maximum);
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder(super.toString());
- if (getNumberOfValues() != 0) {
- buf.append(" min: ");
- buf.append(getMinimum());
- buf.append(" max: ");
- buf.append(getMaximum());
- }
- return buf.toString();
- }
- }
-
- private long count = 0;
- private boolean hasNull = false;
-
- ColumnStatisticsImpl(OrcProto.ColumnStatistics stats) {
- if (stats.hasNumberOfValues()) {
- count = stats.getNumberOfValues();
- }
-
- if (stats.hasHasNull()) {
- hasNull = stats.getHasNull();
- } else {
- hasNull = true;
- }
- }
-
- ColumnStatisticsImpl() {
- }
-
- void increment() {
- count += 1;
- }
-
- void increment(int count) {
- this.count += count;
- }
-
- void setNull() {
- hasNull = true;
- }
-
- void updateBoolean(boolean value, int repetitions) {
- throw new UnsupportedOperationException("Can't update boolean");
- }
-
- void updateInteger(long value, int repetitions) {
- throw new UnsupportedOperationException("Can't update integer");
- }
-
- void updateDouble(double value) {
- throw new UnsupportedOperationException("Can't update double");
- }
-
- void updateString(Text value) {
- throw new UnsupportedOperationException("Can't update string");
- }
-
- void updateString(byte[] bytes, int offset, int length, int repetitions) {
- throw new UnsupportedOperationException("Can't update string");
- }
-
- void updateBinary(BytesWritable value) {
- throw new UnsupportedOperationException("Can't update binary");
- }
-
- void updateBinary(byte[] bytes, int offset, int length, int repetitions) {
- throw new UnsupportedOperationException("Can't update string");
- }
-
- void updateDecimal(HiveDecimal value) {
- throw new UnsupportedOperationException("Can't update decimal");
- }
-
- void updateDate(DateWritable value) {
- throw new UnsupportedOperationException("Can't update date");
- }
-
- void updateDate(int value) {
- throw new UnsupportedOperationException("Can't update date");
- }
-
- void updateTimestamp(Timestamp value) {
- throw new UnsupportedOperationException("Can't update timestamp");
- }
-
- void updateTimestamp(long value) {
- throw new UnsupportedOperationException("Can't update timestamp");
- }
-
- boolean isStatsExists() {
- return (count > 0 || hasNull == true);
- }
-
- void merge(ColumnStatisticsImpl stats) {
- count += stats.count;
- hasNull |= stats.hasNull;
- }
-
- void reset() {
- count = 0;
- hasNull = false;
- }
-
- @Override
- public long getNumberOfValues() {
- return count;
- }
-
- @Override
- public boolean hasNull() {
- return hasNull;
- }
-
- @Override
- public String toString() {
- return "count: " + count + " hasNull: " + hasNull;
- }
-
- OrcProto.ColumnStatistics.Builder serialize() {
- OrcProto.ColumnStatistics.Builder builder =
- OrcProto.ColumnStatistics.newBuilder();
- builder.setNumberOfValues(count);
- builder.setHasNull(hasNull);
- return builder;
- }
-
- static ColumnStatisticsImpl create(TypeDescription schema) {
- switch (schema.getCategory()) {
- case BOOLEAN:
- return new BooleanStatisticsImpl();
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- return new IntegerStatisticsImpl();
- case FLOAT:
- case DOUBLE:
- return new DoubleStatisticsImpl();
- case STRING:
- case CHAR:
- case VARCHAR:
- return new StringStatisticsImpl();
- case DECIMAL:
- return new DecimalStatisticsImpl();
- case DATE:
- return new DateStatisticsImpl();
- case TIMESTAMP:
- return new TimestampStatisticsImpl();
- case BINARY:
- return new BinaryStatisticsImpl();
- default:
- return new ColumnStatisticsImpl();
- }
- }
-
- static ColumnStatisticsImpl deserialize(OrcProto.ColumnStatistics stats) {
- if (stats.hasBucketStatistics()) {
- return new BooleanStatisticsImpl(stats);
- } else if (stats.hasIntStatistics()) {
- return new IntegerStatisticsImpl(stats);
- } else if (stats.hasDoubleStatistics()) {
- return new DoubleStatisticsImpl(stats);
- } else if (stats.hasStringStatistics()) {
- return new StringStatisticsImpl(stats);
- } else if (stats.hasDecimalStatistics()) {
- return new DecimalStatisticsImpl(stats);
- } else if (stats.hasDateStatistics()) {
- return new DateStatisticsImpl(stats);
- } else if (stats.hasTimestampStatistics()) {
- return new TimestampStatisticsImpl(stats);
- } else if(stats.hasBinaryStatistics()) {
- return new BinaryStatisticsImpl(stats);
- } else {
- return new ColumnStatisticsImpl(stats);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java
deleted file mode 100644
index ed9d7ac..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.EnumSet;
-
-import javax.annotation.Nullable;
-
-public interface CompressionCodec {
-
- public enum Modifier {
- /* speed/compression tradeoffs */
- FASTEST,
- FAST,
- DEFAULT,
- /* data sensitivity modifiers */
- TEXT,
- BINARY
- };
-
- /**
- * Compress the in buffer to the out buffer.
- * @param in the bytes to compress
- * @param out the uncompressed bytes
- * @param overflow put any additional bytes here
- * @return true if the output is smaller than input
- * @throws IOException
- */
- boolean compress(ByteBuffer in, ByteBuffer out, ByteBuffer overflow
- ) throws IOException;
-
- /**
- * Decompress the in buffer to the out buffer.
- * @param in the bytes to decompress
- * @param out the decompressed bytes
- * @throws IOException
- */
- void decompress(ByteBuffer in, ByteBuffer out) throws IOException;
-
- /**
- * Produce a modified compression codec if the underlying algorithm allows
- * modification.
- *
- * This does not modify the current object, but returns a new object if
- * modifications are possible. Returns the same object if no modifications
- * are possible.
- * @param modifiers compression modifiers
- * @return codec for use after optional modification
- */
- CompressionCodec modify(@Nullable EnumSet<Modifier> modifiers);
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionKind.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionKind.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionKind.java
index 07c6116..22627df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionKind.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionKind.java
@@ -20,8 +20,22 @@ package org.apache.hadoop.hive.ql.io.orc;
/**
* An enumeration that lists the generic compression algorithms that
- * can be applied to ORC files.
+ * can be applied to ORC files. This is a shim to help users while we
+ * migrate to the org.apache.orc package.
*/
public enum CompressionKind {
- NONE, ZLIB, SNAPPY, LZO
+ NONE(org.apache.orc.CompressionKind.NONE),
+ ZLIB(org.apache.orc.CompressionKind.ZLIB),
+ SNAPPY(org.apache.orc.CompressionKind.SNAPPY),
+ LZO(org.apache.orc.CompressionKind.LZO);
+
+ CompressionKind(org.apache.orc.CompressionKind underlying) {
+ this.underlying = underlying;
+ }
+
+ public org.apache.orc.CompressionKind getUnderlying() {
+ return underlying;
+ }
+
+ private final org.apache.orc.CompressionKind underlying;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DataReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DataReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DataReader.java
deleted file mode 100644
index e0d9943..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DataReader.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hive.common.io.DiskRangeList;
-
-/** An abstract data reader that IO formats can use to read bytes from underlying storage. */
-public interface DataReader {
-
- /** Opens the DataReader, making it ready to use. */
- void open() throws IOException;
-
- /** Closes the DataReader. */
- void close() throws IOException;
-
- /** Reads the data.
- *
- * Note that for the cases such as zero-copy read, caller must release the disk ranges
- * produced after being done with them. Call isTrackingDiskRanges to find out if this is needed.
- * @param range List if disk ranges to read. Ranges with data will be ignored.
- * @param baseOffset Base offset from the start of the file of the ranges in disk range list.
- * @param doForceDirect Whether the data should be read into direct buffers.
- * @return New or modified list of DiskRange-s, where all the ranges are filled with data.
- */
- DiskRangeList readFileData(
- DiskRangeList range, long baseOffset, boolean doForceDirect) throws IOException;
-
-
- /**
- * Whether the user should release buffers created by readFileData. See readFileData javadoc.
- */
- boolean isTrackingDiskRanges();
-
- /**
- * Releases buffers created by readFileData. See readFileData javadoc.
- * @param toRelease The buffer to release.
- */
- void releaseBuffer(ByteBuffer toRelease);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DateColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DateColumnStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DateColumnStatistics.java
deleted file mode 100644
index ae3fe31..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DateColumnStatistics.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.util.Date;
-
-/**
- * Statistics for DATE columns.
- */
-public interface DateColumnStatistics extends ColumnStatistics {
- /**
- * Get the minimum value for the column.
- * @return minimum value
- */
- Date getMinimum();
-
- /**
- * Get the maximum value for the column.
- * @return maximum value
- */
- Date getMaximum();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DecimalColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DecimalColumnStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DecimalColumnStatistics.java
deleted file mode 100644
index ec6aa43..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DecimalColumnStatistics.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-
-/**
- * Statistics for decimal columns.
- */
-public interface DecimalColumnStatistics extends ColumnStatistics {
-
- /**
- * Get the minimum value for the column.
- * @return the minimum value
- */
- HiveDecimal getMinimum();
-
- /**
- * Get the maximum value for the column.
- * @return the maximum value
- */
- HiveDecimal getMaximum();
-
- /**
- * Get the sum of the values of the column.
- * @return the sum
- */
- HiveDecimal getSum();
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DirectDecompressionCodec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DirectDecompressionCodec.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DirectDecompressionCodec.java
deleted file mode 100644
index 41a77b0..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DirectDecompressionCodec.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public interface DirectDecompressionCodec extends CompressionCodec {
- public boolean isAvailable();
- public void directDecompress(ByteBuffer in, ByteBuffer out) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DoubleColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DoubleColumnStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DoubleColumnStatistics.java
deleted file mode 100644
index 6af7535..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DoubleColumnStatistics.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * Statistics for float and double columns.
- */
-public interface DoubleColumnStatistics extends ColumnStatistics {
-
- /**
- * Get the smallest value in the column. Only defined if getNumberOfValues
- * is non-zero.
- * @return the minimum
- */
- double getMinimum();
-
- /**
- * Get the largest value in the column. Only defined if getNumberOfValues
- * is non-zero.
- * @return the maximum
- */
- double getMaximum();
-
- /**
- * Get the sum of the values in the column.
- * @return the sum
- */
- double getSum();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java
deleted file mode 100644
index 063c53c..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.io.Text;
-
-/**
- * A class that is a growable array of bytes. Growth is managed in terms of
- * chunks that are allocated when needed.
- */
-public final class DynamicByteArray {
- static final int DEFAULT_CHUNKSIZE = 32 * 1024;
- static final int DEFAULT_NUM_CHUNKS = 128;
-
- private final int chunkSize; // our allocation sizes
- private byte[][] data; // the real data
- private int length; // max set element index +1
- private int initializedChunks = 0; // the number of chunks created
-
- public DynamicByteArray() {
- this(DEFAULT_NUM_CHUNKS, DEFAULT_CHUNKSIZE);
- }
-
- public DynamicByteArray(int numChunks, int chunkSize) {
- if (chunkSize == 0) {
- throw new IllegalArgumentException("bad chunksize");
- }
- this.chunkSize = chunkSize;
- data = new byte[numChunks][];
- }
-
- /**
- * Ensure that the given index is valid.
- */
- private void grow(int chunkIndex) {
- if (chunkIndex >= initializedChunks) {
- if (chunkIndex >= data.length) {
- int newSize = Math.max(chunkIndex + 1, 2 * data.length);
- byte[][] newChunk = new byte[newSize][];
- System.arraycopy(data, 0, newChunk, 0, data.length);
- data = newChunk;
- }
- for(int i=initializedChunks; i <= chunkIndex; ++i) {
- data[i] = new byte[chunkSize];
- }
- initializedChunks = chunkIndex + 1;
- }
- }
-
- public byte get(int index) {
- if (index >= length) {
- throw new IndexOutOfBoundsException("Index " + index +
- " is outside of 0.." +
- (length - 1));
- }
- int i = index / chunkSize;
- int j = index % chunkSize;
- return data[i][j];
- }
-
- public void set(int index, byte value) {
- int i = index / chunkSize;
- int j = index % chunkSize;
- grow(i);
- if (index >= length) {
- length = index + 1;
- }
- data[i][j] = value;
- }
-
- public int add(byte value) {
- int i = length / chunkSize;
- int j = length % chunkSize;
- grow(i);
- data[i][j] = value;
- int result = length;
- length += 1;
- return result;
- }
-
- /**
- * Copy a slice of a byte array into our buffer.
- * @param value the array to copy from
- * @param valueOffset the first location to copy from value
- * @param valueLength the number of bytes to copy from value
- * @return the offset of the start of the value
- */
- public int add(byte[] value, int valueOffset, int valueLength) {
- int i = length / chunkSize;
- int j = length % chunkSize;
- grow((length + valueLength) / chunkSize);
- int remaining = valueLength;
- while (remaining > 0) {
- int size = Math.min(remaining, chunkSize - j);
- System.arraycopy(value, valueOffset, data[i], j, size);
- remaining -= size;
- valueOffset += size;
- i += 1;
- j = 0;
- }
- int result = length;
- length += valueLength;
- return result;
- }
-
- /**
- * Read the entire stream into this array.
- * @param in the stream to read from
- * @throws IOException
- */
- public void readAll(InputStream in) throws IOException {
- int currentChunk = length / chunkSize;
- int currentOffset = length % chunkSize;
- grow(currentChunk);
- int currentLength = in.read(data[currentChunk], currentOffset,
- chunkSize - currentOffset);
- while (currentLength > 0) {
- length += currentLength;
- currentOffset = length % chunkSize;
- if (currentOffset == 0) {
- currentChunk = length / chunkSize;
- grow(currentChunk);
- }
- currentLength = in.read(data[currentChunk], currentOffset,
- chunkSize - currentOffset);
- }
- }
-
- /**
- * Byte compare a set of bytes against the bytes in this dynamic array.
- * @param other source of the other bytes
- * @param otherOffset start offset in the other array
- * @param otherLength number of bytes in the other array
- * @param ourOffset the offset in our array
- * @param ourLength the number of bytes in our array
- * @return negative for less, 0 for equal, positive for greater
- */
- public int compare(byte[] other, int otherOffset, int otherLength,
- int ourOffset, int ourLength) {
- int currentChunk = ourOffset / chunkSize;
- int currentOffset = ourOffset % chunkSize;
- int maxLength = Math.min(otherLength, ourLength);
- while (maxLength > 0 &&
- other[otherOffset] == data[currentChunk][currentOffset]) {
- otherOffset += 1;
- currentOffset += 1;
- if (currentOffset == chunkSize) {
- currentChunk += 1;
- currentOffset = 0;
- }
- maxLength -= 1;
- }
- if (maxLength == 0) {
- return otherLength - ourLength;
- }
- int otherByte = 0xff & other[otherOffset];
- int ourByte = 0xff & data[currentChunk][currentOffset];
- return otherByte > ourByte ? 1 : -1;
- }
-
- /**
- * Get the size of the array.
- * @return the number of bytes in the array
- */
- public int size() {
- return length;
- }
-
- /**
- * Clear the array to its original pristine state.
- */
- public void clear() {
- length = 0;
- for(int i=0; i < data.length; ++i) {
- data[i] = null;
- }
- initializedChunks = 0;
- }
-
- /**
- * Set a text value from the bytes in this dynamic array.
- * @param result the value to set
- * @param offset the start of the bytes to copy
- * @param length the number of bytes to copy
- */
- public void setText(Text result, int offset, int length) {
- result.clear();
- int currentChunk = offset / chunkSize;
- int currentOffset = offset % chunkSize;
- int currentLength = Math.min(length, chunkSize - currentOffset);
- while (length > 0) {
- result.append(data[currentChunk], currentOffset, currentLength);
- length -= currentLength;
- currentChunk += 1;
- currentOffset = 0;
- currentLength = Math.min(length, chunkSize - currentOffset);
- }
- }
-
- /**
- * Write out a range of this dynamic array to an output stream.
- * @param out the stream to write to
- * @param offset the first offset to write
- * @param length the number of bytes to write
- * @throws IOException
- */
- public void write(OutputStream out, int offset,
- int length) throws IOException {
- int currentChunk = offset / chunkSize;
- int currentOffset = offset % chunkSize;
- while (length > 0) {
- int currentLength = Math.min(length, chunkSize - currentOffset);
- out.write(data[currentChunk], currentOffset, currentLength);
- length -= currentLength;
- currentChunk += 1;
- currentOffset = 0;
- }
- }
-
- @Override
- public String toString() {
- int i;
- StringBuilder sb = new StringBuilder(length * 3);
-
- sb.append('{');
- int l = length - 1;
- for (i=0; i<l; i++) {
- sb.append(Integer.toHexString(get(i)));
- sb.append(',');
- }
- sb.append(get(i));
- sb.append('}');
-
- return sb.toString();
- }
-
- public void setByteBuffer(ByteBuffer result, int offset, int length) {
- result.clear();
- int currentChunk = offset / chunkSize;
- int currentOffset = offset % chunkSize;
- int currentLength = Math.min(length, chunkSize - currentOffset);
- while (length > 0) {
- result.put(data[currentChunk], currentOffset, currentLength);
- length -= currentLength;
- currentChunk += 1;
- currentOffset = 0;
- currentLength = Math.min(length, chunkSize - currentOffset);
- }
- }
-
- /**
- * Gets all the bytes of the array.
- *
- * @return Bytes of the array
- */
- public byte[] get() {
- byte[] result = null;
- if (length > 0) {
- int currentChunk = 0;
- int currentOffset = 0;
- int currentLength = Math.min(length, chunkSize);
- int destOffset = 0;
- result = new byte[length];
- int totalLength = length;
- while (totalLength > 0) {
- System.arraycopy(data[currentChunk], currentOffset, result, destOffset, currentLength);
- destOffset += currentLength;
- totalLength -= currentLength;
- currentChunk += 1;
- currentOffset = 0;
- currentLength = Math.min(totalLength, chunkSize - currentOffset);
- }
- }
- return result;
- }
-
- /**
- * Get the size of the buffers.
- */
- public long getSizeInBytes() {
- return initializedChunks * chunkSize;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicIntArray.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicIntArray.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicIntArray.java
deleted file mode 100644
index 2e884c0..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicIntArray.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * Dynamic int array that uses primitive types and chunks to avoid copying
- * large number of integers when it resizes.
- *
- * The motivation for this class is memory optimization, i.e. space efficient
- * storage of potentially huge arrays without good a-priori size guesses.
- *
- * The API of this class is between a primitive array and a AbstractList. It's
- * not a Collection implementation because it handles primitive types, but the
- * API could be extended to support iterators and the like.
- *
- * NOTE: Like standard Collection implementations/arrays, this class is not
- * synchronized.
- */
-final class DynamicIntArray {
- static final int DEFAULT_CHUNKSIZE = 8 * 1024;
- static final int INIT_CHUNKS = 128;
-
- private final int chunkSize; // our allocation size
- private int[][] data; // the real data
- private int length; // max set element index +1
- private int initializedChunks = 0; // the number of created chunks
-
- public DynamicIntArray() {
- this(DEFAULT_CHUNKSIZE);
- }
-
- public DynamicIntArray(int chunkSize) {
- this.chunkSize = chunkSize;
-
- data = new int[INIT_CHUNKS][];
- }
-
- /**
- * Ensure that the given index is valid.
- */
- private void grow(int chunkIndex) {
- if (chunkIndex >= initializedChunks) {
- if (chunkIndex >= data.length) {
- int newSize = Math.max(chunkIndex + 1, 2 * data.length);
- int[][] newChunk = new int[newSize][];
- System.arraycopy(data, 0, newChunk, 0, data.length);
- data = newChunk;
- }
- for (int i=initializedChunks; i <= chunkIndex; ++i) {
- data[i] = new int[chunkSize];
- }
- initializedChunks = chunkIndex + 1;
- }
- }
-
- public int get(int index) {
- if (index >= length) {
- throw new IndexOutOfBoundsException("Index " + index +
- " is outside of 0.." +
- (length - 1));
- }
- int i = index / chunkSize;
- int j = index % chunkSize;
- return data[i][j];
- }
-
- public void set(int index, int value) {
- int i = index / chunkSize;
- int j = index % chunkSize;
- grow(i);
- if (index >= length) {
- length = index + 1;
- }
- data[i][j] = value;
- }
-
- public void increment(int index, int value) {
- int i = index / chunkSize;
- int j = index % chunkSize;
- grow(i);
- if (index >= length) {
- length = index + 1;
- }
- data[i][j] += value;
- }
-
- public void add(int value) {
- int i = length / chunkSize;
- int j = length % chunkSize;
- grow(i);
- data[i][j] = value;
- length += 1;
- }
-
- public int size() {
- return length;
- }
-
- public void clear() {
- length = 0;
- for(int i=0; i < data.length; ++i) {
- data[i] = null;
- }
- initializedChunks = 0;
- }
-
- public String toString() {
- int i;
- StringBuilder sb = new StringBuilder(length * 4);
-
- sb.append('{');
- int l = length - 1;
- for (i=0; i<l; i++) {
- sb.append(get(i));
- sb.append(',');
- }
- sb.append(get(i));
- sb.append('}');
-
- return sb.toString();
- }
-
- public int getSizeInBytes() {
- return 4 * initializedChunks * chunkSize;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
index f17b154..a242a37 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
@@ -41,8 +41,6 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.filters.BloomFilterIO;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
@@ -50,6 +48,13 @@ import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.impl.ColumnStatisticsImpl;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.OrcProto;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONWriter;
@@ -381,7 +386,7 @@ public final class FileDump {
for (int colIdx : rowIndexCols) {
sargColumns[colIdx] = true;
}
- RecordReaderImpl.Index indices = rows
+ OrcIndex indices = rows
.readRowIndex(stripeIx, null, null, null, sargColumns);
for (int col : rowIndexCols) {
StringBuilder buf = new StringBuilder();
@@ -649,9 +654,10 @@ public final class FileDump {
return sb.toString();
}
- private static String getFormattedRowIndices(int col, RowIndex[] rowGroupIndex) {
+ private static String getFormattedRowIndices(int col,
+ OrcProto.RowIndex[] rowGroupIndex) {
StringBuilder buf = new StringBuilder();
- RowIndex index;
+ OrcProto.RowIndex index;
buf.append(" Row group indices for column ").append(col).append(":");
if (rowGroupIndex == null || (col >= rowGroupIndex.length) ||
((index = rowGroupIndex[col]) == null)) {
@@ -661,7 +667,7 @@ public final class FileDump {
for (int entryIx = 0; entryIx < index.getEntryCount(); ++entryIx) {
buf.append("\n Entry ").append(entryIx).append(": ");
- RowIndexEntry entry = index.getEntry(entryIx);
+ OrcProto.RowIndexEntry entry = index.getEntry(entryIx);
if (entry == null) {
buf.append("unknown\n");
continue;
@@ -686,7 +692,7 @@ public final class FileDump {
public static long getTotalPaddingSize(Reader reader) throws IOException {
long paddedBytes = 0;
- List<org.apache.hadoop.hive.ql.io.orc.StripeInformation> stripes = reader.getStripes();
+ List<StripeInformation> stripes = reader.getStripes();
for (int i = 1; i < stripes.size(); i++) {
long prevStripeOffset = stripes.get(i - 1).getOffset();
long prevStripeLen = stripes.get(i - 1).getLength();
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileMetaInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileMetaInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileMetaInfo.java
deleted file mode 100644
index 95c674e..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileMetaInfo.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.WriterVersion;
-
-/**
- * FileMetaInfo - represents file metadata stored in footer and postscript sections of the file
- * that is useful for Reader implementation
- *
- */
-public class FileMetaInfo {
- ByteBuffer footerMetaAndPsBuffer;
- final String compressionType;
- final int bufferSize;
- final int metadataSize;
- final ByteBuffer footerBuffer;
- final List<Integer> versionList;
- final OrcFile.WriterVersion writerVersion;
-
-
- /** Ctor used when reading splits - no version list or full footer buffer. */
- FileMetaInfo(String compressionType, int bufferSize, int metadataSize,
- ByteBuffer footerBuffer, OrcFile.WriterVersion writerVersion) {
- this(compressionType, bufferSize, metadataSize, footerBuffer, null,
- writerVersion, null);
- }
-
- /** Ctor used when creating file info during init and when getting a new one. */
- public FileMetaInfo(String compressionType, int bufferSize, int metadataSize,
- ByteBuffer footerBuffer, List<Integer> versionList, WriterVersion writerVersion,
- ByteBuffer fullFooterBuffer) {
- this.compressionType = compressionType;
- this.bufferSize = bufferSize;
- this.metadataSize = metadataSize;
- this.footerBuffer = footerBuffer;
- this.versionList = versionList;
- this.writerVersion = writerVersion;
- this.footerMetaAndPsBuffer = fullFooterBuffer;
- }
-
- public OrcFile.WriterVersion getWriterVersion() {
- return writerVersion;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileMetadata.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileMetadata.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileMetadata.java
deleted file mode 100644
index 26b27a3..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileMetadata.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.ql.io.orc.OrcProto;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
-
-/**
- * Cached file metadata. Right now, it caches everything; we don't have to store all the
- * protobuf structs actually, we could just store what we need, but that would require that
- * ORC stop depending on them too. Luckily, they shouldn't be very big.
- */
-public interface FileMetadata {
- boolean isOriginalFormat();
-
- List<StripeInformation> getStripes();
-
- CompressionKind getCompressionKind();
-
- int getCompressionBufferSize();
-
- int getRowIndexStride();
-
- int getColumnCount();
-
- int getFlattenedColumnCount();
-
- long getFileId();
-
- List<Integer> getVersionList();
-
- int getMetadataSize();
-
- int getWriterVersionNum();
-
- List<Type> getTypes();
-
- List<OrcProto.StripeStatistics> getStripeStats();
-
- long getContentLength();
-
- long getNumberOfRows();
-
- List<OrcProto.ColumnStatistics> getFileStats();
-}
\ No newline at end of file
[04/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java
deleted file mode 100644
index 9e878f5..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-
-/**
- * A reader that reads a sequence of integers.
- * */
-public class RunLengthIntegerReader implements IntegerReader {
- private InStream input;
- private final boolean signed;
- private final long[] literals =
- new long[RunLengthIntegerWriter.MAX_LITERAL_SIZE];
- private int numLiterals = 0;
- private int delta = 0;
- private int used = 0;
- private boolean repeat = false;
- private SerializationUtils utils;
-
- public RunLengthIntegerReader(InStream input, boolean signed) throws IOException {
- this.input = input;
- this.signed = signed;
- this.utils = new SerializationUtils();
- }
-
- private void readValues(boolean ignoreEof) throws IOException {
- int control = input.read();
- if (control == -1) {
- if (!ignoreEof) {
- throw new EOFException("Read past end of RLE integer from " + input);
- }
- used = numLiterals = 0;
- return;
- } else if (control < 0x80) {
- numLiterals = control + RunLengthIntegerWriter.MIN_REPEAT_SIZE;
- used = 0;
- repeat = true;
- delta = input.read();
- if (delta == -1) {
- throw new EOFException("End of stream in RLE Integer from " + input);
- }
- // convert from 0 to 255 to -128 to 127 by converting to a signed byte
- delta = (byte) (0 + delta);
- if (signed) {
- literals[0] = utils.readVslong(input);
- } else {
- literals[0] = utils.readVulong(input);
- }
- } else {
- repeat = false;
- numLiterals = 0x100 - control;
- used = 0;
- for(int i=0; i < numLiterals; ++i) {
- if (signed) {
- literals[i] = utils.readVslong(input);
- } else {
- literals[i] = utils.readVulong(input);
- }
- }
- }
- }
-
- @Override
- public boolean hasNext() throws IOException {
- return used != numLiterals || input.available() > 0;
- }
-
- @Override
- public long next() throws IOException {
- long result;
- if (used == numLiterals) {
- readValues(false);
- }
- if (repeat) {
- result = literals[0] + (used++) * delta;
- } else {
- result = literals[used++];
- }
- return result;
- }
-
- @Override
- public void nextVector(LongColumnVector previous, long previousLen) throws IOException {
- previous.isRepeating = true;
- for (int i = 0; i < previousLen; i++) {
- if (!previous.isNull[i]) {
- previous.vector[i] = next();
- } else {
- // The default value of null for int type in vectorized
- // processing is 1, so set that if the value is null
- previous.vector[i] = 1;
- }
-
- // The default value for nulls in Vectorization for int types is 1
- // and given that non null value can also be 1, we need to check for isNull also
- // when determining the isRepeating flag.
- if (previous.isRepeating
- && i > 0
- && (previous.vector[i - 1] != previous.vector[i] || previous.isNull[i - 1] != previous.isNull[i])) {
- previous.isRepeating = false;
- }
- }
- }
-
- @Override
- public void setInStream(InStream data) {
- input = data;
- }
-
- @Override
- public void seek(PositionProvider index) throws IOException {
- input.seek(index);
- int consumed = (int) index.getNext();
- if (consumed != 0) {
- // a loop is required for cases where we break the run into two parts
- while (consumed > 0) {
- readValues(false);
- used = consumed;
- consumed -= numLiterals;
- }
- } else {
- used = 0;
- numLiterals = 0;
- }
- }
-
- @Override
- public void skip(long numValues) throws IOException {
- while (numValues > 0) {
- if (used == numLiterals) {
- readValues(false);
- }
- long consume = Math.min(numValues, numLiterals - used);
- used += consume;
- numValues -= consume;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java
deleted file mode 100644
index aef3231..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java
+++ /dev/null
@@ -1,392 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.ErrorMsg;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.io.orc.RunLengthIntegerWriterV2.EncodingType;
-
-/**
- * A reader that reads a sequence of light weight compressed integers. Refer
- * {@link RunLengthIntegerWriterV2} for description of various lightweight
- * compression techniques.
- */
-public class RunLengthIntegerReaderV2 implements IntegerReader {
- public static final Logger LOG = LoggerFactory.getLogger(RunLengthIntegerReaderV2.class);
-
- private InStream input;
- private final boolean signed;
- private final long[] literals = new long[RunLengthIntegerWriterV2.MAX_SCOPE];
- private boolean isRepeating = false;
- private int numLiterals = 0;
- private int used = 0;
- private final boolean skipCorrupt;
- private final SerializationUtils utils;
- private EncodingType currentEncoding;
-
- public RunLengthIntegerReaderV2(InStream input, boolean signed,
- boolean skipCorrupt) throws IOException {
- this.input = input;
- this.signed = signed;
- this.skipCorrupt = skipCorrupt;
- this.utils = new SerializationUtils();
- }
-
- private final static EncodingType[] encodings = EncodingType.values();
- private void readValues(boolean ignoreEof) throws IOException {
- // read the first 2 bits and determine the encoding type
- isRepeating = false;
- int firstByte = input.read();
- if (firstByte < 0) {
- if (!ignoreEof) {
- throw new EOFException("Read past end of RLE integer from " + input);
- }
- used = numLiterals = 0;
- return;
- }
- currentEncoding = encodings[(firstByte >>> 6) & 0x03];
- switch (currentEncoding) {
- case SHORT_REPEAT: readShortRepeatValues(firstByte); break;
- case DIRECT: readDirectValues(firstByte); break;
- case PATCHED_BASE: readPatchedBaseValues(firstByte); break;
- case DELTA: readDeltaValues(firstByte); break;
- default: throw new IOException("Unknown encoding " + currentEncoding);
- }
- }
-
- private void readDeltaValues(int firstByte) throws IOException {
-
- // extract the number of fixed bits
- int fb = (firstByte >>> 1) & 0x1f;
- if (fb != 0) {
- fb = utils.decodeBitWidth(fb);
- }
-
- // extract the blob run length
- int len = (firstByte & 0x01) << 8;
- len |= input.read();
-
- // read the first value stored as vint
- long firstVal = 0;
- if (signed) {
- firstVal = utils.readVslong(input);
- } else {
- firstVal = utils.readVulong(input);
- }
-
- // store first value to result buffer
- long prevVal = firstVal;
- literals[numLiterals++] = firstVal;
-
- // if fixed bits is 0 then all values have fixed delta
- if (fb == 0) {
- // read the fixed delta value stored as vint (deltas can be negative even
- // if all number are positive)
- long fd = utils.readVslong(input);
- if (fd == 0) {
- isRepeating = true;
- assert numLiterals == 1;
- Arrays.fill(literals, numLiterals, numLiterals + len, literals[0]);
- numLiterals += len;
- } else {
- // add fixed deltas to adjacent values
- for(int i = 0; i < len; i++) {
- literals[numLiterals++] = literals[numLiterals - 2] + fd;
- }
- }
- } else {
- long deltaBase = utils.readVslong(input);
- // add delta base and first value
- literals[numLiterals++] = firstVal + deltaBase;
- prevVal = literals[numLiterals - 1];
- len -= 1;
-
- // write the unpacked values, add it to previous value and store final
- // value to result buffer. if the delta base value is negative then it
- // is a decreasing sequence else an increasing sequence
- utils.readInts(literals, numLiterals, len, fb, input);
- while (len > 0) {
- if (deltaBase < 0) {
- literals[numLiterals] = prevVal - literals[numLiterals];
- } else {
- literals[numLiterals] = prevVal + literals[numLiterals];
- }
- prevVal = literals[numLiterals];
- len--;
- numLiterals++;
- }
- }
- }
-
- private void readPatchedBaseValues(int firstByte) throws IOException {
-
- // extract the number of fixed bits
- int fbo = (firstByte >>> 1) & 0x1f;
- int fb = utils.decodeBitWidth(fbo);
-
- // extract the run length of data blob
- int len = (firstByte & 0x01) << 8;
- len |= input.read();
- // runs are always one off
- len += 1;
-
- // extract the number of bytes occupied by base
- int thirdByte = input.read();
- int bw = (thirdByte >>> 5) & 0x07;
- // base width is one off
- bw += 1;
-
- // extract patch width
- int pwo = thirdByte & 0x1f;
- int pw = utils.decodeBitWidth(pwo);
-
- // read fourth byte and extract patch gap width
- int fourthByte = input.read();
- int pgw = (fourthByte >>> 5) & 0x07;
- // patch gap width is one off
- pgw += 1;
-
- // extract the length of the patch list
- int pl = fourthByte & 0x1f;
-
- // read the next base width number of bytes to extract base value
- long base = utils.bytesToLongBE(input, bw);
- long mask = (1L << ((bw * 8) - 1));
- // if MSB of base value is 1 then base is negative value else positive
- if ((base & mask) != 0) {
- base = base & ~mask;
- base = -base;
- }
-
- // unpack the data blob
- long[] unpacked = new long[len];
- utils.readInts(unpacked, 0, len, fb, input);
-
- // unpack the patch blob
- long[] unpackedPatch = new long[pl];
-
- if ((pw + pgw) > 64 && !skipCorrupt) {
- throw new IOException("Corruption in ORC data encountered. To skip" +
- " reading corrupted data, set hive.exec.orc.skip.corrupt.data to" +
- " true");
- }
- int bitSize = utils.getClosestFixedBits(pw + pgw);
- utils.readInts(unpackedPatch, 0, pl, bitSize, input);
-
- // apply the patch directly when decoding the packed data
- int patchIdx = 0;
- long currGap = 0;
- long currPatch = 0;
- long patchMask = ((1L << pw) - 1);
- currGap = unpackedPatch[patchIdx] >>> pw;
- currPatch = unpackedPatch[patchIdx] & patchMask;
- long actualGap = 0;
-
- // special case: gap is >255 then patch value will be 0.
- // if gap is <=255 then patch value cannot be 0
- while (currGap == 255 && currPatch == 0) {
- actualGap += 255;
- patchIdx++;
- currGap = unpackedPatch[patchIdx] >>> pw;
- currPatch = unpackedPatch[patchIdx] & patchMask;
- }
- // add the left over gap
- actualGap += currGap;
-
- // unpack data blob, patch it (if required), add base to get final result
- for(int i = 0; i < unpacked.length; i++) {
- if (i == actualGap) {
- // extract the patch value
- long patchedVal = unpacked[i] | (currPatch << fb);
-
- // add base to patched value
- literals[numLiterals++] = base + patchedVal;
-
- // increment the patch to point to next entry in patch list
- patchIdx++;
-
- if (patchIdx < pl) {
- // read the next gap and patch
- currGap = unpackedPatch[patchIdx] >>> pw;
- currPatch = unpackedPatch[patchIdx] & patchMask;
- actualGap = 0;
-
- // special case: gap is >255 then patch will be 0. if gap is
- // <=255 then patch cannot be 0
- while (currGap == 255 && currPatch == 0) {
- actualGap += 255;
- patchIdx++;
- currGap = unpackedPatch[patchIdx] >>> pw;
- currPatch = unpackedPatch[patchIdx] & patchMask;
- }
- // add the left over gap
- actualGap += currGap;
-
- // next gap is relative to the current gap
- actualGap += i;
- }
- } else {
- // no patching required. add base to unpacked value to get final value
- literals[numLiterals++] = base + unpacked[i];
- }
- }
-
- }
-
- private void readDirectValues(int firstByte) throws IOException {
-
- // extract the number of fixed bits
- int fbo = (firstByte >>> 1) & 0x1f;
- int fb = utils.decodeBitWidth(fbo);
-
- // extract the run length
- int len = (firstByte & 0x01) << 8;
- len |= input.read();
- // runs are one off
- len += 1;
-
- // write the unpacked values and zigzag decode to result buffer
- utils.readInts(literals, numLiterals, len, fb, input);
- if (signed) {
- for(int i = 0; i < len; i++) {
- literals[numLiterals] = utils.zigzagDecode(literals[numLiterals]);
- numLiterals++;
- }
- } else {
- numLiterals += len;
- }
- }
-
- private void readShortRepeatValues(int firstByte) throws IOException {
-
- // read the number of bytes occupied by the value
- int size = (firstByte >>> 3) & 0x07;
- // #bytes are one off
- size += 1;
-
- // read the run length
- int len = firstByte & 0x07;
- // run lengths values are stored only after MIN_REPEAT value is met
- len += RunLengthIntegerWriterV2.MIN_REPEAT;
-
- // read the repeated value which is store using fixed bytes
- long val = utils.bytesToLongBE(input, size);
-
- if (signed) {
- val = utils.zigzagDecode(val);
- }
-
- if (numLiterals != 0) {
- // Currently this always holds, which makes peekNextAvailLength simpler.
- // If this changes, peekNextAvailLength should be adjusted accordingly.
- throw new AssertionError("readValues called with existing values present");
- }
- // repeat the value for length times
- isRepeating = true;
- // TODO: this is not so useful and V1 reader doesn't do that. Fix? Same if delta == 0
- for(int i = 0; i < len; i++) {
- literals[i] = val;
- }
- numLiterals = len;
- }
-
- @Override
- public boolean hasNext() throws IOException {
- return used != numLiterals || input.available() > 0;
- }
-
- @Override
- public long next() throws IOException {
- long result;
- if (used == numLiterals) {
- numLiterals = 0;
- used = 0;
- readValues(false);
- }
- result = literals[used++];
- return result;
- }
-
- @Override
- public void seek(PositionProvider index) throws IOException {
- input.seek(index);
- int consumed = (int) index.getNext();
- if (consumed != 0) {
- // a loop is required for cases where we break the run into two
- // parts
- while (consumed > 0) {
- numLiterals = 0;
- readValues(false);
- used = consumed;
- consumed -= numLiterals;
- }
- } else {
- used = 0;
- numLiterals = 0;
- }
- }
-
- @Override
- public void skip(long numValues) throws IOException {
- while (numValues > 0) {
- if (used == numLiterals) {
- numLiterals = 0;
- used = 0;
- readValues(false);
- }
- long consume = Math.min(numValues, numLiterals - used);
- used += consume;
- numValues -= consume;
- }
- }
-
- @Override
- public void nextVector(LongColumnVector previous, long previousLen) throws IOException {
- previous.isRepeating = true;
- for (int i = 0; i < previousLen; i++) {
- if (!previous.isNull[i]) {
- previous.vector[i] = next();
- } else {
- // The default value of null for int type in vectorized
- // processing is 1, so set that if the value is null
- previous.vector[i] = 1;
- }
-
- // The default value for nulls in Vectorization for int types is 1
- // and given that non null value can also be 1, we need to check for isNull also
- // when determining the isRepeating flag.
- if (previous.isRepeating
- && i > 0
- && (previous.vector[i - 1] != previous.vector[i] ||
- previous.isNull[i - 1] != previous.isNull[i])) {
- previous.isRepeating = false;
- }
- }
- }
-
- @Override
- public void setInStream(InStream data) {
- input = data;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java
deleted file mode 100644
index 078eae8..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-
-/**
- * A streamFactory that writes a sequence of integers. A control byte is written before
- * each run with positive values 0 to 127 meaning 3 to 130 repetitions, each
- * repetition is offset by a delta. If the control byte is -1 to -128, 1 to 128
- * literal vint values follow.
- */
-class RunLengthIntegerWriter implements IntegerWriter {
- static final int MIN_REPEAT_SIZE = 3;
- static final int MAX_DELTA = 127;
- static final int MIN_DELTA = -128;
- static final int MAX_LITERAL_SIZE = 128;
- private static final int MAX_REPEAT_SIZE = 127 + MIN_REPEAT_SIZE;
- private final PositionedOutputStream output;
- private final boolean signed;
- private final long[] literals = new long[MAX_LITERAL_SIZE];
- private int numLiterals = 0;
- private long delta = 0;
- private boolean repeat = false;
- private int tailRunLength = 0;
- private SerializationUtils utils;
-
- RunLengthIntegerWriter(PositionedOutputStream output,
- boolean signed) {
- this.output = output;
- this.signed = signed;
- this.utils = new SerializationUtils();
- }
-
- private void writeValues() throws IOException {
- if (numLiterals != 0) {
- if (repeat) {
- output.write(numLiterals - MIN_REPEAT_SIZE);
- output.write((byte) delta);
- if (signed) {
- utils.writeVslong(output, literals[0]);
- } else {
- utils.writeVulong(output, literals[0]);
- }
- } else {
- output.write(-numLiterals);
- for(int i=0; i < numLiterals; ++i) {
- if (signed) {
- utils.writeVslong(output, literals[i]);
- } else {
- utils.writeVulong(output, literals[i]);
- }
- }
- }
- repeat = false;
- numLiterals = 0;
- tailRunLength = 0;
- }
- }
-
- @Override
- public void flush() throws IOException {
- writeValues();
- output.flush();
- }
-
- @Override
- public void write(long value) throws IOException {
- if (numLiterals == 0) {
- literals[numLiterals++] = value;
- tailRunLength = 1;
- } else if (repeat) {
- if (value == literals[0] + delta * numLiterals) {
- numLiterals += 1;
- if (numLiterals == MAX_REPEAT_SIZE) {
- writeValues();
- }
- } else {
- writeValues();
- literals[numLiterals++] = value;
- tailRunLength = 1;
- }
- } else {
- if (tailRunLength == 1) {
- delta = value - literals[numLiterals - 1];
- if (delta < MIN_DELTA || delta > MAX_DELTA) {
- tailRunLength = 1;
- } else {
- tailRunLength = 2;
- }
- } else if (value == literals[numLiterals - 1] + delta) {
- tailRunLength += 1;
- } else {
- delta = value - literals[numLiterals - 1];
- if (delta < MIN_DELTA || delta > MAX_DELTA) {
- tailRunLength = 1;
- } else {
- tailRunLength = 2;
- }
- }
- if (tailRunLength == MIN_REPEAT_SIZE) {
- if (numLiterals + 1 == MIN_REPEAT_SIZE) {
- repeat = true;
- numLiterals += 1;
- } else {
- numLiterals -= MIN_REPEAT_SIZE - 1;
- long base = literals[numLiterals];
- writeValues();
- literals[0] = base;
- repeat = true;
- numLiterals = MIN_REPEAT_SIZE;
- }
- } else {
- literals[numLiterals++] = value;
- if (numLiterals == MAX_LITERAL_SIZE) {
- writeValues();
- }
- }
- }
- }
-
- @Override
- public void getPosition(PositionRecorder recorder) throws IOException {
- output.getPosition(recorder);
- recorder.addPosition(numLiterals);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
deleted file mode 100644
index 95f8cc8..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java
+++ /dev/null
@@ -1,831 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-
-/**
- * A writer that performs light weight compression over sequence of integers.
- * <p>
- * There are four types of lightweight integer compression
- * <ul>
- * <li>SHORT_REPEAT</li>
- * <li>DIRECT</li>
- * <li>PATCHED_BASE</li>
- * <li>DELTA</li>
- * </ul>
- * </p>
- * The description and format for these types are as below:
- * <p>
- * <b>SHORT_REPEAT:</b> Used for short repeated integer sequences.
- * <ul>
- * <li>1 byte header
- * <ul>
- * <li>2 bits for encoding type</li>
- * <li>3 bits for bytes required for repeating value</li>
- * <li>3 bits for repeat count (MIN_REPEAT + run length)</li>
- * </ul>
- * </li>
- * <li>Blob - repeat value (fixed bytes)</li>
- * </ul>
- * </p>
- * <p>
- * <b>DIRECT:</b> Used for random integer sequences whose number of bit
- * requirement doesn't vary a lot.
- * <ul>
- * <li>2 bytes header
- * <ul>
- * 1st byte
- * <li>2 bits for encoding type</li>
- * <li>5 bits for fixed bit width of values in blob</li>
- * <li>1 bit for storing MSB of run length</li>
- * </ul>
- * <ul>
- * 2nd byte
- * <li>8 bits for lower run length bits</li>
- * </ul>
- * </li>
- * <li>Blob - stores the direct values using fixed bit width. The length of the
- * data blob is (fixed width * run length) bits long</li>
- * </ul>
- * </p>
- * <p>
- * <b>PATCHED_BASE:</b> Used for random integer sequences whose number of bit
- * requirement varies beyond a threshold.
- * <ul>
- * <li>4 bytes header
- * <ul>
- * 1st byte
- * <li>2 bits for encoding type</li>
- * <li>5 bits for fixed bit width of values in blob</li>
- * <li>1 bit for storing MSB of run length</li>
- * </ul>
- * <ul>
- * 2nd byte
- * <li>8 bits for lower run length bits</li>
- * </ul>
- * <ul>
- * 3rd byte
- * <li>3 bits for bytes required to encode base value</li>
- * <li>5 bits for patch width</li>
- * </ul>
- * <ul>
- * 4th byte
- * <li>3 bits for patch gap width</li>
- * <li>5 bits for patch length</li>
- * </ul>
- * </li>
- * <li>Base value - Stored using fixed number of bytes. If MSB is set, base
- * value is negative else positive. Length of base value is (base width * 8)
- * bits.</li>
- * <li>Data blob - Base reduced values as stored using fixed bit width. Length
- * of data blob is (fixed width * run length) bits.</li>
- * <li>Patch blob - Patch blob is a list of gap and patch value. Each entry in
- * the patch list is (patch width + patch gap width) bits long. Gap between the
- * subsequent elements to be patched are stored in upper part of entry whereas
- * patch values are stored in lower part of entry. Length of patch blob is
- * ((patch width + patch gap width) * patch length) bits.</li>
- * </ul>
- * </p>
- * <p>
- * <b>DELTA</b> Used for monotonically increasing or decreasing sequences,
- * sequences with fixed delta values or long repeated sequences.
- * <ul>
- * <li>2 bytes header
- * <ul>
- * 1st byte
- * <li>2 bits for encoding type</li>
- * <li>5 bits for fixed bit width of values in blob</li>
- * <li>1 bit for storing MSB of run length</li>
- * </ul>
- * <ul>
- * 2nd byte
- * <li>8 bits for lower run length bits</li>
- * </ul>
- * </li>
- * <li>Base value - zigzag encoded value written as varint</li>
- * <li>Delta base - zigzag encoded value written as varint</li>
- * <li>Delta blob - only positive values. monotonicity and orderness are decided
- * based on the sign of the base value and delta base</li>
- * </ul>
- * </p>
- */
-class RunLengthIntegerWriterV2 implements IntegerWriter {
-
- public enum EncodingType {
- SHORT_REPEAT, DIRECT, PATCHED_BASE, DELTA
- }
-
- static final int MAX_SCOPE = 512;
- static final int MIN_REPEAT = 3;
- private static final int MAX_SHORT_REPEAT_LENGTH = 10;
- private long prevDelta = 0;
- private int fixedRunLength = 0;
- private int variableRunLength = 0;
- private final long[] literals = new long[MAX_SCOPE];
- private final PositionedOutputStream output;
- private final boolean signed;
- private EncodingType encoding;
- private int numLiterals;
- private final long[] zigzagLiterals = new long[MAX_SCOPE];
- private final long[] baseRedLiterals = new long[MAX_SCOPE];
- private final long[] adjDeltas = new long[MAX_SCOPE];
- private long fixedDelta;
- private int zzBits90p;
- private int zzBits100p;
- private int brBits95p;
- private int brBits100p;
- private int bitsDeltaMax;
- private int patchWidth;
- private int patchGapWidth;
- private int patchLength;
- private long[] gapVsPatchList;
- private long min;
- private boolean isFixedDelta;
- private SerializationUtils utils;
- private boolean alignedBitpacking;
-
- RunLengthIntegerWriterV2(PositionedOutputStream output, boolean signed) {
- this(output, signed, true);
- }
-
- RunLengthIntegerWriterV2(PositionedOutputStream output, boolean signed,
- boolean alignedBitpacking) {
- this.output = output;
- this.signed = signed;
- this.alignedBitpacking = alignedBitpacking;
- this.utils = new SerializationUtils();
- clear();
- }
-
- private void writeValues() throws IOException {
- if (numLiterals != 0) {
-
- if (encoding.equals(EncodingType.SHORT_REPEAT)) {
- writeShortRepeatValues();
- } else if (encoding.equals(EncodingType.DIRECT)) {
- writeDirectValues();
- } else if (encoding.equals(EncodingType.PATCHED_BASE)) {
- writePatchedBaseValues();
- } else {
- writeDeltaValues();
- }
-
- // clear all the variables
- clear();
- }
- }
-
- private void writeDeltaValues() throws IOException {
- int len = 0;
- int fb = bitsDeltaMax;
- int efb = 0;
-
- if (alignedBitpacking) {
- fb = utils.getClosestAlignedFixedBits(fb);
- }
-
- if (isFixedDelta) {
- // if fixed run length is greater than threshold then it will be fixed
- // delta sequence with delta value 0 else fixed delta sequence with
- // non-zero delta value
- if (fixedRunLength > MIN_REPEAT) {
- // ex. sequence: 2 2 2 2 2 2 2 2
- len = fixedRunLength - 1;
- fixedRunLength = 0;
- } else {
- // ex. sequence: 4 6 8 10 12 14 16
- len = variableRunLength - 1;
- variableRunLength = 0;
- }
- } else {
- // fixed width 0 is used for long repeating values.
- // sequences that require only 1 bit to encode will have an additional bit
- if (fb == 1) {
- fb = 2;
- }
- efb = utils.encodeBitWidth(fb);
- efb = efb << 1;
- len = variableRunLength - 1;
- variableRunLength = 0;
- }
-
- // extract the 9th bit of run length
- final int tailBits = (len & 0x100) >>> 8;
-
- // create first byte of the header
- final int headerFirstByte = getOpcode() | efb | tailBits;
-
- // second byte of the header stores the remaining 8 bits of runlength
- final int headerSecondByte = len & 0xff;
-
- // write header
- output.write(headerFirstByte);
- output.write(headerSecondByte);
-
- // store the first value from zigzag literal array
- if (signed) {
- utils.writeVslong(output, literals[0]);
- } else {
- utils.writeVulong(output, literals[0]);
- }
-
- if (isFixedDelta) {
- // if delta is fixed then we don't need to store delta blob
- utils.writeVslong(output, fixedDelta);
- } else {
- // store the first value as delta value using zigzag encoding
- utils.writeVslong(output, adjDeltas[0]);
-
- // adjacent delta values are bit packed. The length of adjDeltas array is
- // always one less than the number of literals (delta difference for n
- // elements is n-1). We have already written one element, write the
- // remaining numLiterals - 2 elements here
- utils.writeInts(adjDeltas, 1, numLiterals - 2, fb, output);
- }
- }
-
- private void writePatchedBaseValues() throws IOException {
-
- // NOTE: Aligned bit packing cannot be applied for PATCHED_BASE encoding
- // because patch is applied to MSB bits. For example: If fixed bit width of
- // base value is 7 bits and if patch is 3 bits, the actual value is
- // constructed by shifting the patch to left by 7 positions.
- // actual_value = patch << 7 | base_value
- // So, if we align base_value then actual_value can not be reconstructed.
-
- // write the number of fixed bits required in next 5 bits
- final int fb = brBits95p;
- final int efb = utils.encodeBitWidth(fb) << 1;
-
- // adjust variable run length, they are one off
- variableRunLength -= 1;
-
- // extract the 9th bit of run length
- final int tailBits = (variableRunLength & 0x100) >>> 8;
-
- // create first byte of the header
- final int headerFirstByte = getOpcode() | efb | tailBits;
-
- // second byte of the header stores the remaining 8 bits of runlength
- final int headerSecondByte = variableRunLength & 0xff;
-
- // if the min value is negative toggle the sign
- final boolean isNegative = min < 0 ? true : false;
- if (isNegative) {
- min = -min;
- }
-
- // find the number of bytes required for base and shift it by 5 bits
- // to accommodate patch width. The additional bit is used to store the sign
- // of the base value.
- final int baseWidth = utils.findClosestNumBits(min) + 1;
- final int baseBytes = baseWidth % 8 == 0 ? baseWidth / 8 : (baseWidth / 8) + 1;
- final int bb = (baseBytes - 1) << 5;
-
- // if the base value is negative then set MSB to 1
- if (isNegative) {
- min |= (1L << ((baseBytes * 8) - 1));
- }
-
- // third byte contains 3 bits for number of bytes occupied by base
- // and 5 bits for patchWidth
- final int headerThirdByte = bb | utils.encodeBitWidth(patchWidth);
-
- // fourth byte contains 3 bits for page gap width and 5 bits for
- // patch length
- final int headerFourthByte = (patchGapWidth - 1) << 5 | patchLength;
-
- // write header
- output.write(headerFirstByte);
- output.write(headerSecondByte);
- output.write(headerThirdByte);
- output.write(headerFourthByte);
-
- // write the base value using fixed bytes in big endian order
- for(int i = baseBytes - 1; i >= 0; i--) {
- byte b = (byte) ((min >>> (i * 8)) & 0xff);
- output.write(b);
- }
-
- // base reduced literals are bit packed
- int closestFixedBits = utils.getClosestFixedBits(fb);
-
- utils.writeInts(baseRedLiterals, 0, numLiterals, closestFixedBits,
- output);
-
- // write patch list
- closestFixedBits = utils.getClosestFixedBits(patchGapWidth + patchWidth);
-
- utils.writeInts(gapVsPatchList, 0, gapVsPatchList.length, closestFixedBits,
- output);
-
- // reset run length
- variableRunLength = 0;
- }
-
- /**
- * Store the opcode in 2 MSB bits
- * @return opcode
- */
- private int getOpcode() {
- return encoding.ordinal() << 6;
- }
-
- private void writeDirectValues() throws IOException {
-
- // write the number of fixed bits required in next 5 bits
- int fb = zzBits100p;
-
- if (alignedBitpacking) {
- fb = utils.getClosestAlignedFixedBits(fb);
- }
-
- final int efb = utils.encodeBitWidth(fb) << 1;
-
- // adjust variable run length
- variableRunLength -= 1;
-
- // extract the 9th bit of run length
- final int tailBits = (variableRunLength & 0x100) >>> 8;
-
- // create first byte of the header
- final int headerFirstByte = getOpcode() | efb | tailBits;
-
- // second byte of the header stores the remaining 8 bits of runlength
- final int headerSecondByte = variableRunLength & 0xff;
-
- // write header
- output.write(headerFirstByte);
- output.write(headerSecondByte);
-
- // bit packing the zigzag encoded literals
- utils.writeInts(zigzagLiterals, 0, numLiterals, fb, output);
-
- // reset run length
- variableRunLength = 0;
- }
-
- private void writeShortRepeatValues() throws IOException {
- // get the value that is repeating, compute the bits and bytes required
- long repeatVal = 0;
- if (signed) {
- repeatVal = utils.zigzagEncode(literals[0]);
- } else {
- repeatVal = literals[0];
- }
-
- final int numBitsRepeatVal = utils.findClosestNumBits(repeatVal);
- final int numBytesRepeatVal = numBitsRepeatVal % 8 == 0 ? numBitsRepeatVal >>> 3
- : (numBitsRepeatVal >>> 3) + 1;
-
- // write encoding type in top 2 bits
- int header = getOpcode();
-
- // write the number of bytes required for the value
- header |= ((numBytesRepeatVal - 1) << 3);
-
- // write the run length
- fixedRunLength -= MIN_REPEAT;
- header |= fixedRunLength;
-
- // write the header
- output.write(header);
-
- // write the repeating value in big endian byte order
- for(int i = numBytesRepeatVal - 1; i >= 0; i--) {
- int b = (int) ((repeatVal >>> (i * 8)) & 0xff);
- output.write(b);
- }
-
- fixedRunLength = 0;
- }
-
- private void determineEncoding() {
-
- // we need to compute zigzag values for DIRECT encoding if we decide to
- // break early for delta overflows or for shorter runs
- computeZigZagLiterals();
-
- zzBits100p = utils.percentileBits(zigzagLiterals, 0, numLiterals, 1.0);
-
- // not a big win for shorter runs to determine encoding
- if (numLiterals <= MIN_REPEAT) {
- encoding = EncodingType.DIRECT;
- return;
- }
-
- // DELTA encoding check
-
- // for identifying monotonic sequences
- boolean isIncreasing = true;
- boolean isDecreasing = true;
- this.isFixedDelta = true;
-
- this.min = literals[0];
- long max = literals[0];
- final long initialDelta = literals[1] - literals[0];
- long currDelta = initialDelta;
- long deltaMax = initialDelta;
- this.adjDeltas[0] = initialDelta;
-
- for (int i = 1; i < numLiterals; i++) {
- final long l1 = literals[i];
- final long l0 = literals[i - 1];
- currDelta = l1 - l0;
- min = Math.min(min, l1);
- max = Math.max(max, l1);
-
- isIncreasing &= (l0 <= l1);
- isDecreasing &= (l0 >= l1);
-
- isFixedDelta &= (currDelta == initialDelta);
- if (i > 1) {
- adjDeltas[i - 1] = Math.abs(currDelta);
- deltaMax = Math.max(deltaMax, adjDeltas[i - 1]);
- }
- }
-
- // its faster to exit under delta overflow condition without checking for
- // PATCHED_BASE condition as encoding using DIRECT is faster and has less
- // overhead than PATCHED_BASE
- if (!utils.isSafeSubtract(max, min)) {
- encoding = EncodingType.DIRECT;
- return;
- }
-
- // invariant - subtracting any number from any other in the literals after
- // this point won't overflow
-
- // if min is equal to max then the delta is 0, this condition happens for
- // fixed values run >10 which cannot be encoded with SHORT_REPEAT
- if (min == max) {
- assert isFixedDelta : min + "==" + max +
- ", isFixedDelta cannot be false";
- assert currDelta == 0 : min + "==" + max + ", currDelta should be zero";
- fixedDelta = 0;
- encoding = EncodingType.DELTA;
- return;
- }
-
- if (isFixedDelta) {
- assert currDelta == initialDelta
- : "currDelta should be equal to initialDelta for fixed delta encoding";
- encoding = EncodingType.DELTA;
- fixedDelta = currDelta;
- return;
- }
-
- // if initialDelta is 0 then we cannot delta encode as we cannot identify
- // the sign of deltas (increasing or decreasing)
- if (initialDelta != 0) {
- // stores the number of bits required for packing delta blob in
- // delta encoding
- bitsDeltaMax = utils.findClosestNumBits(deltaMax);
-
- // monotonic condition
- if (isIncreasing || isDecreasing) {
- encoding = EncodingType.DELTA;
- return;
- }
- }
-
- // PATCHED_BASE encoding check
-
- // percentile values are computed for the zigzag encoded values. if the
- // number of bit requirement between 90th and 100th percentile varies
- // beyond a threshold then we need to patch the values. if the variation
- // is not significant then we can use direct encoding
-
- zzBits90p = utils.percentileBits(zigzagLiterals, 0, numLiterals, 0.9);
- int diffBitsLH = zzBits100p - zzBits90p;
-
- // if the difference between 90th percentile and 100th percentile fixed
- // bits is > 1 then we need patch the values
- if (diffBitsLH > 1) {
-
- // patching is done only on base reduced values.
- // remove base from literals
- for (int i = 0; i < numLiterals; i++) {
- baseRedLiterals[i] = literals[i] - min;
- }
-
- // 95th percentile width is used to determine max allowed value
- // after which patching will be done
- brBits95p = utils.percentileBits(baseRedLiterals, 0, numLiterals, 0.95);
-
- // 100th percentile is used to compute the max patch width
- brBits100p = utils.percentileBits(baseRedLiterals, 0, numLiterals, 1.0);
-
- // after base reducing the values, if the difference in bits between
- // 95th percentile and 100th percentile value is zero then there
- // is no point in patching the values, in which case we will
- // fallback to DIRECT encoding.
- // The decision to use patched base was based on zigzag values, but the
- // actual patching is done on base reduced literals.
- if ((brBits100p - brBits95p) != 0) {
- encoding = EncodingType.PATCHED_BASE;
- preparePatchedBlob();
- return;
- } else {
- encoding = EncodingType.DIRECT;
- return;
- }
- } else {
- // if difference in bits between 95th percentile and 100th percentile is
- // 0, then patch length will become 0. Hence we will fallback to direct
- encoding = EncodingType.DIRECT;
- return;
- }
- }
-
- private void computeZigZagLiterals() {
- // populate zigzag encoded literals
- long zzEncVal = 0;
- for (int i = 0; i < numLiterals; i++) {
- if (signed) {
- zzEncVal = utils.zigzagEncode(literals[i]);
- } else {
- zzEncVal = literals[i];
- }
- zigzagLiterals[i] = zzEncVal;
- }
- }
-
- private void preparePatchedBlob() {
- // mask will be max value beyond which patch will be generated
- long mask = (1L << brBits95p) - 1;
-
- // since we are considering only 95 percentile, the size of gap and
- // patch array can contain only be 5% values
- patchLength = (int) Math.ceil((numLiterals * 0.05));
-
- int[] gapList = new int[patchLength];
- long[] patchList = new long[patchLength];
-
- // #bit for patch
- patchWidth = brBits100p - brBits95p;
- patchWidth = utils.getClosestFixedBits(patchWidth);
-
- // if patch bit requirement is 64 then it will not possible to pack
- // gap and patch together in a long. To make sure gap and patch can be
- // packed together adjust the patch width
- if (patchWidth == 64) {
- patchWidth = 56;
- brBits95p = 8;
- mask = (1L << brBits95p) - 1;
- }
-
- int gapIdx = 0;
- int patchIdx = 0;
- int prev = 0;
- int gap = 0;
- int maxGap = 0;
-
- for(int i = 0; i < numLiterals; i++) {
- // if value is above mask then create the patch and record the gap
- if (baseRedLiterals[i] > mask) {
- gap = i - prev;
- if (gap > maxGap) {
- maxGap = gap;
- }
-
- // gaps are relative, so store the previous patched value index
- prev = i;
- gapList[gapIdx++] = gap;
-
- // extract the most significant bits that are over mask bits
- long patch = baseRedLiterals[i] >>> brBits95p;
- patchList[patchIdx++] = patch;
-
- // strip off the MSB to enable safe bit packing
- baseRedLiterals[i] &= mask;
- }
- }
-
- // adjust the patch length to number of entries in gap list
- patchLength = gapIdx;
-
- // if the element to be patched is the first and only element then
- // max gap will be 0, but to store the gap as 0 we need atleast 1 bit
- if (maxGap == 0 && patchLength != 0) {
- patchGapWidth = 1;
- } else {
- patchGapWidth = utils.findClosestNumBits(maxGap);
- }
-
- // special case: if the patch gap width is greater than 256, then
- // we need 9 bits to encode the gap width. But we only have 3 bits in
- // header to record the gap width. To deal with this case, we will save
- // two entries in patch list in the following way
- // 256 gap width => 0 for patch value
- // actual gap - 256 => actual patch value
- // We will do the same for gap width = 511. If the element to be patched is
- // the last element in the scope then gap width will be 511. In this case we
- // will have 3 entries in the patch list in the following way
- // 255 gap width => 0 for patch value
- // 255 gap width => 0 for patch value
- // 1 gap width => actual patch value
- if (patchGapWidth > 8) {
- patchGapWidth = 8;
- // for gap = 511, we need two additional entries in patch list
- if (maxGap == 511) {
- patchLength += 2;
- } else {
- patchLength += 1;
- }
- }
-
- // create gap vs patch list
- gapIdx = 0;
- patchIdx = 0;
- gapVsPatchList = new long[patchLength];
- for(int i = 0; i < patchLength; i++) {
- long g = gapList[gapIdx++];
- long p = patchList[patchIdx++];
- while (g > 255) {
- gapVsPatchList[i++] = (255L << patchWidth);
- g -= 255;
- }
-
- // store patch value in LSBs and gap in MSBs
- gapVsPatchList[i] = (g << patchWidth) | p;
- }
- }
-
- /**
- * clears all the variables
- */
- private void clear() {
- numLiterals = 0;
- encoding = null;
- prevDelta = 0;
- fixedDelta = 0;
- zzBits90p = 0;
- zzBits100p = 0;
- brBits95p = 0;
- brBits100p = 0;
- bitsDeltaMax = 0;
- patchGapWidth = 0;
- patchLength = 0;
- patchWidth = 0;
- gapVsPatchList = null;
- min = 0;
- isFixedDelta = true;
- }
-
- @Override
- public void flush() throws IOException {
- if (numLiterals != 0) {
- if (variableRunLength != 0) {
- determineEncoding();
- writeValues();
- } else if (fixedRunLength != 0) {
- if (fixedRunLength < MIN_REPEAT) {
- variableRunLength = fixedRunLength;
- fixedRunLength = 0;
- determineEncoding();
- writeValues();
- } else if (fixedRunLength >= MIN_REPEAT
- && fixedRunLength <= MAX_SHORT_REPEAT_LENGTH) {
- encoding = EncodingType.SHORT_REPEAT;
- writeValues();
- } else {
- encoding = EncodingType.DELTA;
- isFixedDelta = true;
- writeValues();
- }
- }
- }
- output.flush();
- }
-
- @Override
- public void write(long val) throws IOException {
- if (numLiterals == 0) {
- initializeLiterals(val);
- } else {
- if (numLiterals == 1) {
- prevDelta = val - literals[0];
- literals[numLiterals++] = val;
- // if both values are same count as fixed run else variable run
- if (val == literals[0]) {
- fixedRunLength = 2;
- variableRunLength = 0;
- } else {
- fixedRunLength = 0;
- variableRunLength = 2;
- }
- } else {
- long currentDelta = val - literals[numLiterals - 1];
- if (prevDelta == 0 && currentDelta == 0) {
- // fixed delta run
-
- literals[numLiterals++] = val;
-
- // if variable run is non-zero then we are seeing repeating
- // values at the end of variable run in which case keep
- // updating variable and fixed runs
- if (variableRunLength > 0) {
- fixedRunLength = 2;
- }
- fixedRunLength += 1;
-
- // if fixed run met the minimum condition and if variable
- // run is non-zero then flush the variable run and shift the
- // tail fixed runs to start of the buffer
- if (fixedRunLength >= MIN_REPEAT && variableRunLength > 0) {
- numLiterals -= MIN_REPEAT;
- variableRunLength -= MIN_REPEAT - 1;
- // copy the tail fixed runs
- long[] tailVals = new long[MIN_REPEAT];
- System.arraycopy(literals, numLiterals, tailVals, 0, MIN_REPEAT);
-
- // determine variable encoding and flush values
- determineEncoding();
- writeValues();
-
- // shift tail fixed runs to beginning of the buffer
- for(long l : tailVals) {
- literals[numLiterals++] = l;
- }
- }
-
- // if fixed runs reached max repeat length then write values
- if (fixedRunLength == MAX_SCOPE) {
- determineEncoding();
- writeValues();
- }
- } else {
- // variable delta run
-
- // if fixed run length is non-zero and if it satisfies the
- // short repeat conditions then write the values as short repeats
- // else use delta encoding
- if (fixedRunLength >= MIN_REPEAT) {
- if (fixedRunLength <= MAX_SHORT_REPEAT_LENGTH) {
- encoding = EncodingType.SHORT_REPEAT;
- writeValues();
- } else {
- encoding = EncodingType.DELTA;
- isFixedDelta = true;
- writeValues();
- }
- }
-
- // if fixed run length is <MIN_REPEAT and current value is
- // different from previous then treat it as variable run
- if (fixedRunLength > 0 && fixedRunLength < MIN_REPEAT) {
- if (val != literals[numLiterals - 1]) {
- variableRunLength = fixedRunLength;
- fixedRunLength = 0;
- }
- }
-
- // after writing values re-initialize the variables
- if (numLiterals == 0) {
- initializeLiterals(val);
- } else {
- // keep updating variable run lengths
- prevDelta = val - literals[numLiterals - 1];
- literals[numLiterals++] = val;
- variableRunLength += 1;
-
- // if variable run length reach the max scope, write it
- if (variableRunLength == MAX_SCOPE) {
- determineEncoding();
- writeValues();
- }
- }
- }
- }
- }
- }
-
- private void initializeLiterals(long val) {
- literals[numLiterals++] = val;
- fixedRunLength = 1;
- variableRunLength = 1;
- }
-
- @Override
- public void getPosition(PositionRecorder recorder) throws IOException {
- output.getPosition(recorder);
- recorder.addPosition(numLiterals);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SchemaEvolution.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SchemaEvolution.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SchemaEvolution.java
index 9d00eb2..f28ca13 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SchemaEvolution.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SchemaEvolution.java
@@ -24,8 +24,10 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
import org.apache.hadoop.hive.ql.io.orc.TreeReaderFactory.TreeReaderSchema;
+import org.apache.orc.OrcProto;
+import org.apache.orc.OrcUtils;
+import org.apache.orc.TypeDescription;
/**
* Take the file types and the (optional) configuration column names/types and see if there
@@ -96,7 +98,7 @@ public class SchemaEvolution {
}
}
- List<Type> fullSchemaTypes;
+ List<OrcProto.Type> fullSchemaTypes;
if (isAcid) {
fullSchemaTypes = new ArrayList<OrcProto.Type>();
@@ -140,9 +142,9 @@ public class SchemaEvolution {
* @param typeDescr
* @return ORC types for the ACID event based on the row's type description
*/
- public static List<Type> createEventSchema(TypeDescription typeDescr) {
+ public static List<OrcProto.Type> createEventSchema(TypeDescription typeDescr) {
- List<Type> result = new ArrayList<Type>();
+ List<OrcProto.Type> result = new ArrayList<OrcProto.Type>();
OrcProto.Type.Builder type = OrcProto.Type.newBuilder();
type.setKind(OrcProto.Type.Kind.STRUCT);
@@ -182,4 +184,4 @@ public class SchemaEvolution {
acidEventOrcTypeKinds.add(OrcProto.Type.Kind.LONG);
acidEventOrcTypeKinds.add(OrcProto.Type.Kind.STRUCT);
}
-}
\ No newline at end of file
+}
[11/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/RunLengthIntegerReaderV2.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/RunLengthIntegerReaderV2.java b/orc/src/java/org/apache/orc/impl/RunLengthIntegerReaderV2.java
new file mode 100644
index 0000000..5f2a673
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/RunLengthIntegerReaderV2.java
@@ -0,0 +1,390 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+
+/**
+ * A reader that reads a sequence of light weight compressed integers. Refer
+ * {@link RunLengthIntegerWriterV2} for description of various lightweight
+ * compression techniques.
+ */
+public class RunLengthIntegerReaderV2 implements IntegerReader {
+ public static final Logger LOG = LoggerFactory.getLogger(RunLengthIntegerReaderV2.class);
+
+ private InStream input;
+ private final boolean signed;
+ private final long[] literals = new long[RunLengthIntegerWriterV2.MAX_SCOPE];
+ private boolean isRepeating = false;
+ private int numLiterals = 0;
+ private int used = 0;
+ private final boolean skipCorrupt;
+ private final SerializationUtils utils;
+ private RunLengthIntegerWriterV2.EncodingType currentEncoding;
+
+ public RunLengthIntegerReaderV2(InStream input, boolean signed,
+ boolean skipCorrupt) throws IOException {
+ this.input = input;
+ this.signed = signed;
+ this.skipCorrupt = skipCorrupt;
+ this.utils = new SerializationUtils();
+ }
+
+ private final static RunLengthIntegerWriterV2.EncodingType[] encodings = RunLengthIntegerWriterV2.EncodingType.values();
+ private void readValues(boolean ignoreEof) throws IOException {
+ // read the first 2 bits and determine the encoding type
+ isRepeating = false;
+ int firstByte = input.read();
+ if (firstByte < 0) {
+ if (!ignoreEof) {
+ throw new EOFException("Read past end of RLE integer from " + input);
+ }
+ used = numLiterals = 0;
+ return;
+ }
+ currentEncoding = encodings[(firstByte >>> 6) & 0x03];
+ switch (currentEncoding) {
+ case SHORT_REPEAT: readShortRepeatValues(firstByte); break;
+ case DIRECT: readDirectValues(firstByte); break;
+ case PATCHED_BASE: readPatchedBaseValues(firstByte); break;
+ case DELTA: readDeltaValues(firstByte); break;
+ default: throw new IOException("Unknown encoding " + currentEncoding);
+ }
+ }
+
+ private void readDeltaValues(int firstByte) throws IOException {
+
+ // extract the number of fixed bits
+ int fb = (firstByte >>> 1) & 0x1f;
+ if (fb != 0) {
+ fb = utils.decodeBitWidth(fb);
+ }
+
+ // extract the blob run length
+ int len = (firstByte & 0x01) << 8;
+ len |= input.read();
+
+ // read the first value stored as vint
+ long firstVal = 0;
+ if (signed) {
+ firstVal = utils.readVslong(input);
+ } else {
+ firstVal = utils.readVulong(input);
+ }
+
+ // store first value to result buffer
+ long prevVal = firstVal;
+ literals[numLiterals++] = firstVal;
+
+ // if fixed bits is 0 then all values have fixed delta
+ if (fb == 0) {
+ // read the fixed delta value stored as vint (deltas can be negative even
+ // if all number are positive)
+ long fd = utils.readVslong(input);
+ if (fd == 0) {
+ isRepeating = true;
+ assert numLiterals == 1;
+ Arrays.fill(literals, numLiterals, numLiterals + len, literals[0]);
+ numLiterals += len;
+ } else {
+ // add fixed deltas to adjacent values
+ for(int i = 0; i < len; i++) {
+ literals[numLiterals++] = literals[numLiterals - 2] + fd;
+ }
+ }
+ } else {
+ long deltaBase = utils.readVslong(input);
+ // add delta base and first value
+ literals[numLiterals++] = firstVal + deltaBase;
+ prevVal = literals[numLiterals - 1];
+ len -= 1;
+
+ // write the unpacked values, add it to previous value and store final
+ // value to result buffer. if the delta base value is negative then it
+ // is a decreasing sequence else an increasing sequence
+ utils.readInts(literals, numLiterals, len, fb, input);
+ while (len > 0) {
+ if (deltaBase < 0) {
+ literals[numLiterals] = prevVal - literals[numLiterals];
+ } else {
+ literals[numLiterals] = prevVal + literals[numLiterals];
+ }
+ prevVal = literals[numLiterals];
+ len--;
+ numLiterals++;
+ }
+ }
+ }
+
+ private void readPatchedBaseValues(int firstByte) throws IOException {
+
+ // extract the number of fixed bits
+ int fbo = (firstByte >>> 1) & 0x1f;
+ int fb = utils.decodeBitWidth(fbo);
+
+ // extract the run length of data blob
+ int len = (firstByte & 0x01) << 8;
+ len |= input.read();
+ // runs are always one off
+ len += 1;
+
+ // extract the number of bytes occupied by base
+ int thirdByte = input.read();
+ int bw = (thirdByte >>> 5) & 0x07;
+ // base width is one off
+ bw += 1;
+
+ // extract patch width
+ int pwo = thirdByte & 0x1f;
+ int pw = utils.decodeBitWidth(pwo);
+
+ // read fourth byte and extract patch gap width
+ int fourthByte = input.read();
+ int pgw = (fourthByte >>> 5) & 0x07;
+ // patch gap width is one off
+ pgw += 1;
+
+ // extract the length of the patch list
+ int pl = fourthByte & 0x1f;
+
+ // read the next base width number of bytes to extract base value
+ long base = utils.bytesToLongBE(input, bw);
+ long mask = (1L << ((bw * 8) - 1));
+ // if MSB of base value is 1 then base is negative value else positive
+ if ((base & mask) != 0) {
+ base = base & ~mask;
+ base = -base;
+ }
+
+ // unpack the data blob
+ long[] unpacked = new long[len];
+ utils.readInts(unpacked, 0, len, fb, input);
+
+ // unpack the patch blob
+ long[] unpackedPatch = new long[pl];
+
+ if ((pw + pgw) > 64 && !skipCorrupt) {
+ throw new IOException("Corruption in ORC data encountered. To skip" +
+ " reading corrupted data, set hive.exec.orc.skip.corrupt.data to" +
+ " true");
+ }
+ int bitSize = utils.getClosestFixedBits(pw + pgw);
+ utils.readInts(unpackedPatch, 0, pl, bitSize, input);
+
+ // apply the patch directly when decoding the packed data
+ int patchIdx = 0;
+ long currGap = 0;
+ long currPatch = 0;
+ long patchMask = ((1L << pw) - 1);
+ currGap = unpackedPatch[patchIdx] >>> pw;
+ currPatch = unpackedPatch[patchIdx] & patchMask;
+ long actualGap = 0;
+
+ // special case: gap is >255 then patch value will be 0.
+ // if gap is <=255 then patch value cannot be 0
+ while (currGap == 255 && currPatch == 0) {
+ actualGap += 255;
+ patchIdx++;
+ currGap = unpackedPatch[patchIdx] >>> pw;
+ currPatch = unpackedPatch[patchIdx] & patchMask;
+ }
+ // add the left over gap
+ actualGap += currGap;
+
+ // unpack data blob, patch it (if required), add base to get final result
+ for(int i = 0; i < unpacked.length; i++) {
+ if (i == actualGap) {
+ // extract the patch value
+ long patchedVal = unpacked[i] | (currPatch << fb);
+
+ // add base to patched value
+ literals[numLiterals++] = base + patchedVal;
+
+ // increment the patch to point to next entry in patch list
+ patchIdx++;
+
+ if (patchIdx < pl) {
+ // read the next gap and patch
+ currGap = unpackedPatch[patchIdx] >>> pw;
+ currPatch = unpackedPatch[patchIdx] & patchMask;
+ actualGap = 0;
+
+ // special case: gap is >255 then patch will be 0. if gap is
+ // <=255 then patch cannot be 0
+ while (currGap == 255 && currPatch == 0) {
+ actualGap += 255;
+ patchIdx++;
+ currGap = unpackedPatch[patchIdx] >>> pw;
+ currPatch = unpackedPatch[patchIdx] & patchMask;
+ }
+ // add the left over gap
+ actualGap += currGap;
+
+ // next gap is relative to the current gap
+ actualGap += i;
+ }
+ } else {
+ // no patching required. add base to unpacked value to get final value
+ literals[numLiterals++] = base + unpacked[i];
+ }
+ }
+
+ }
+
+ private void readDirectValues(int firstByte) throws IOException {
+
+ // extract the number of fixed bits
+ int fbo = (firstByte >>> 1) & 0x1f;
+ int fb = utils.decodeBitWidth(fbo);
+
+ // extract the run length
+ int len = (firstByte & 0x01) << 8;
+ len |= input.read();
+ // runs are one off
+ len += 1;
+
+ // write the unpacked values and zigzag decode to result buffer
+ utils.readInts(literals, numLiterals, len, fb, input);
+ if (signed) {
+ for(int i = 0; i < len; i++) {
+ literals[numLiterals] = utils.zigzagDecode(literals[numLiterals]);
+ numLiterals++;
+ }
+ } else {
+ numLiterals += len;
+ }
+ }
+
+ private void readShortRepeatValues(int firstByte) throws IOException {
+
+ // read the number of bytes occupied by the value
+ int size = (firstByte >>> 3) & 0x07;
+ // #bytes are one off
+ size += 1;
+
+ // read the run length
+ int len = firstByte & 0x07;
+ // run lengths values are stored only after MIN_REPEAT value is met
+ len += RunLengthIntegerWriterV2.MIN_REPEAT;
+
+ // read the repeated value which is store using fixed bytes
+ long val = utils.bytesToLongBE(input, size);
+
+ if (signed) {
+ val = utils.zigzagDecode(val);
+ }
+
+ if (numLiterals != 0) {
+ // Currently this always holds, which makes peekNextAvailLength simpler.
+ // If this changes, peekNextAvailLength should be adjusted accordingly.
+ throw new AssertionError("readValues called with existing values present");
+ }
+ // repeat the value for length times
+ isRepeating = true;
+ // TODO: this is not so useful and V1 reader doesn't do that. Fix? Same if delta == 0
+ for(int i = 0; i < len; i++) {
+ literals[i] = val;
+ }
+ numLiterals = len;
+ }
+
+ @Override
+ public boolean hasNext() throws IOException {
+ return used != numLiterals || input.available() > 0;
+ }
+
+ @Override
+ public long next() throws IOException {
+ long result;
+ if (used == numLiterals) {
+ numLiterals = 0;
+ used = 0;
+ readValues(false);
+ }
+ result = literals[used++];
+ return result;
+ }
+
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ input.seek(index);
+ int consumed = (int) index.getNext();
+ if (consumed != 0) {
+ // a loop is required for cases where we break the run into two
+ // parts
+ while (consumed > 0) {
+ numLiterals = 0;
+ readValues(false);
+ used = consumed;
+ consumed -= numLiterals;
+ }
+ } else {
+ used = 0;
+ numLiterals = 0;
+ }
+ }
+
+ @Override
+ public void skip(long numValues) throws IOException {
+ while (numValues > 0) {
+ if (used == numLiterals) {
+ numLiterals = 0;
+ used = 0;
+ readValues(false);
+ }
+ long consume = Math.min(numValues, numLiterals - used);
+ used += consume;
+ numValues -= consume;
+ }
+ }
+
+ @Override
+ public void nextVector(LongColumnVector previous, long previousLen) throws IOException {
+ previous.isRepeating = true;
+ for (int i = 0; i < previousLen; i++) {
+ if (!previous.isNull[i]) {
+ previous.vector[i] = next();
+ } else {
+ // The default value of null for int type in vectorized
+ // processing is 1, so set that if the value is null
+ previous.vector[i] = 1;
+ }
+
+ // The default value for nulls in Vectorization for int types is 1
+ // and given that non null value can also be 1, we need to check for isNull also
+ // when determining the isRepeating flag.
+ if (previous.isRepeating
+ && i > 0
+ && (previous.vector[i - 1] != previous.vector[i] ||
+ previous.isNull[i - 1] != previous.isNull[i])) {
+ previous.isRepeating = false;
+ }
+ }
+ }
+
+ @Override
+ public void setInStream(InStream data) {
+ input = data;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/RunLengthIntegerWriter.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/RunLengthIntegerWriter.java b/orc/src/java/org/apache/orc/impl/RunLengthIntegerWriter.java
new file mode 100644
index 0000000..3e5f2e2
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/RunLengthIntegerWriter.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+
+/**
+ * A streamFactory that writes a sequence of integers. A control byte is written before
+ * each run with positive values 0 to 127 meaning 3 to 130 repetitions, each
+ * repetition is offset by a delta. If the control byte is -1 to -128, 1 to 128
+ * literal vint values follow.
+ */
+public class RunLengthIntegerWriter implements IntegerWriter {
+ static final int MIN_REPEAT_SIZE = 3;
+ static final int MAX_DELTA = 127;
+ static final int MIN_DELTA = -128;
+ static final int MAX_LITERAL_SIZE = 128;
+ private static final int MAX_REPEAT_SIZE = 127 + MIN_REPEAT_SIZE;
+ private final PositionedOutputStream output;
+ private final boolean signed;
+ private final long[] literals = new long[MAX_LITERAL_SIZE];
+ private int numLiterals = 0;
+ private long delta = 0;
+ private boolean repeat = false;
+ private int tailRunLength = 0;
+ private SerializationUtils utils;
+
+ public RunLengthIntegerWriter(PositionedOutputStream output,
+ boolean signed) {
+ this.output = output;
+ this.signed = signed;
+ this.utils = new SerializationUtils();
+ }
+
+ private void writeValues() throws IOException {
+ if (numLiterals != 0) {
+ if (repeat) {
+ output.write(numLiterals - MIN_REPEAT_SIZE);
+ output.write((byte) delta);
+ if (signed) {
+ utils.writeVslong(output, literals[0]);
+ } else {
+ utils.writeVulong(output, literals[0]);
+ }
+ } else {
+ output.write(-numLiterals);
+ for(int i=0; i < numLiterals; ++i) {
+ if (signed) {
+ utils.writeVslong(output, literals[i]);
+ } else {
+ utils.writeVulong(output, literals[i]);
+ }
+ }
+ }
+ repeat = false;
+ numLiterals = 0;
+ tailRunLength = 0;
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ writeValues();
+ output.flush();
+ }
+
+ @Override
+ public void write(long value) throws IOException {
+ if (numLiterals == 0) {
+ literals[numLiterals++] = value;
+ tailRunLength = 1;
+ } else if (repeat) {
+ if (value == literals[0] + delta * numLiterals) {
+ numLiterals += 1;
+ if (numLiterals == MAX_REPEAT_SIZE) {
+ writeValues();
+ }
+ } else {
+ writeValues();
+ literals[numLiterals++] = value;
+ tailRunLength = 1;
+ }
+ } else {
+ if (tailRunLength == 1) {
+ delta = value - literals[numLiterals - 1];
+ if (delta < MIN_DELTA || delta > MAX_DELTA) {
+ tailRunLength = 1;
+ } else {
+ tailRunLength = 2;
+ }
+ } else if (value == literals[numLiterals - 1] + delta) {
+ tailRunLength += 1;
+ } else {
+ delta = value - literals[numLiterals - 1];
+ if (delta < MIN_DELTA || delta > MAX_DELTA) {
+ tailRunLength = 1;
+ } else {
+ tailRunLength = 2;
+ }
+ }
+ if (tailRunLength == MIN_REPEAT_SIZE) {
+ if (numLiterals + 1 == MIN_REPEAT_SIZE) {
+ repeat = true;
+ numLiterals += 1;
+ } else {
+ numLiterals -= MIN_REPEAT_SIZE - 1;
+ long base = literals[numLiterals];
+ writeValues();
+ literals[0] = base;
+ repeat = true;
+ numLiterals = MIN_REPEAT_SIZE;
+ }
+ } else {
+ literals[numLiterals++] = value;
+ if (numLiterals == MAX_LITERAL_SIZE) {
+ writeValues();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void getPosition(PositionRecorder recorder) throws IOException {
+ output.getPosition(recorder);
+ recorder.addPosition(numLiterals);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/RunLengthIntegerWriterV2.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/RunLengthIntegerWriterV2.java b/orc/src/java/org/apache/orc/impl/RunLengthIntegerWriterV2.java
new file mode 100644
index 0000000..fab2801
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/RunLengthIntegerWriterV2.java
@@ -0,0 +1,831 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+
+/**
+ * A writer that performs light weight compression over sequence of integers.
+ * <p>
+ * There are four types of lightweight integer compression
+ * <ul>
+ * <li>SHORT_REPEAT</li>
+ * <li>DIRECT</li>
+ * <li>PATCHED_BASE</li>
+ * <li>DELTA</li>
+ * </ul>
+ * </p>
+ * The description and format for these types are as below:
+ * <p>
+ * <b>SHORT_REPEAT:</b> Used for short repeated integer sequences.
+ * <ul>
+ * <li>1 byte header
+ * <ul>
+ * <li>2 bits for encoding type</li>
+ * <li>3 bits for bytes required for repeating value</li>
+ * <li>3 bits for repeat count (MIN_REPEAT + run length)</li>
+ * </ul>
+ * </li>
+ * <li>Blob - repeat value (fixed bytes)</li>
+ * </ul>
+ * </p>
+ * <p>
+ * <b>DIRECT:</b> Used for random integer sequences whose number of bit
+ * requirement doesn't vary a lot.
+ * <ul>
+ * <li>2 bytes header
+ * <ul>
+ * 1st byte
+ * <li>2 bits for encoding type</li>
+ * <li>5 bits for fixed bit width of values in blob</li>
+ * <li>1 bit for storing MSB of run length</li>
+ * </ul>
+ * <ul>
+ * 2nd byte
+ * <li>8 bits for lower run length bits</li>
+ * </ul>
+ * </li>
+ * <li>Blob - stores the direct values using fixed bit width. The length of the
+ * data blob is (fixed width * run length) bits long</li>
+ * </ul>
+ * </p>
+ * <p>
+ * <b>PATCHED_BASE:</b> Used for random integer sequences whose number of bit
+ * requirement varies beyond a threshold.
+ * <ul>
+ * <li>4 bytes header
+ * <ul>
+ * 1st byte
+ * <li>2 bits for encoding type</li>
+ * <li>5 bits for fixed bit width of values in blob</li>
+ * <li>1 bit for storing MSB of run length</li>
+ * </ul>
+ * <ul>
+ * 2nd byte
+ * <li>8 bits for lower run length bits</li>
+ * </ul>
+ * <ul>
+ * 3rd byte
+ * <li>3 bits for bytes required to encode base value</li>
+ * <li>5 bits for patch width</li>
+ * </ul>
+ * <ul>
+ * 4th byte
+ * <li>3 bits for patch gap width</li>
+ * <li>5 bits for patch length</li>
+ * </ul>
+ * </li>
+ * <li>Base value - Stored using fixed number of bytes. If MSB is set, base
+ * value is negative else positive. Length of base value is (base width * 8)
+ * bits.</li>
+ * <li>Data blob - Base reduced values as stored using fixed bit width. Length
+ * of data blob is (fixed width * run length) bits.</li>
+ * <li>Patch blob - Patch blob is a list of gap and patch value. Each entry in
+ * the patch list is (patch width + patch gap width) bits long. Gap between the
+ * subsequent elements to be patched are stored in upper part of entry whereas
+ * patch values are stored in lower part of entry. Length of patch blob is
+ * ((patch width + patch gap width) * patch length) bits.</li>
+ * </ul>
+ * </p>
+ * <p>
+ * <b>DELTA</b> Used for monotonically increasing or decreasing sequences,
+ * sequences with fixed delta values or long repeated sequences.
+ * <ul>
+ * <li>2 bytes header
+ * <ul>
+ * 1st byte
+ * <li>2 bits for encoding type</li>
+ * <li>5 bits for fixed bit width of values in blob</li>
+ * <li>1 bit for storing MSB of run length</li>
+ * </ul>
+ * <ul>
+ * 2nd byte
+ * <li>8 bits for lower run length bits</li>
+ * </ul>
+ * </li>
+ * <li>Base value - zigzag encoded value written as varint</li>
+ * <li>Delta base - zigzag encoded value written as varint</li>
+ * <li>Delta blob - only positive values. monotonicity and orderness are decided
+ * based on the sign of the base value and delta base</li>
+ * </ul>
+ * </p>
+ */
+public class RunLengthIntegerWriterV2 implements IntegerWriter {
+
+ public enum EncodingType {
+ SHORT_REPEAT, DIRECT, PATCHED_BASE, DELTA
+ }
+
+ static final int MAX_SCOPE = 512;
+ static final int MIN_REPEAT = 3;
+ private static final int MAX_SHORT_REPEAT_LENGTH = 10;
+ private long prevDelta = 0;
+ private int fixedRunLength = 0;
+ private int variableRunLength = 0;
+ private final long[] literals = new long[MAX_SCOPE];
+ private final PositionedOutputStream output;
+ private final boolean signed;
+ private EncodingType encoding;
+ private int numLiterals;
+ private final long[] zigzagLiterals = new long[MAX_SCOPE];
+ private final long[] baseRedLiterals = new long[MAX_SCOPE];
+ private final long[] adjDeltas = new long[MAX_SCOPE];
+ private long fixedDelta;
+ private int zzBits90p;
+ private int zzBits100p;
+ private int brBits95p;
+ private int brBits100p;
+ private int bitsDeltaMax;
+ private int patchWidth;
+ private int patchGapWidth;
+ private int patchLength;
+ private long[] gapVsPatchList;
+ private long min;
+ private boolean isFixedDelta;
+ private SerializationUtils utils;
+ private boolean alignedBitpacking;
+
+ RunLengthIntegerWriterV2(PositionedOutputStream output, boolean signed) {
+ this(output, signed, true);
+ }
+
+ public RunLengthIntegerWriterV2(PositionedOutputStream output, boolean signed,
+ boolean alignedBitpacking) {
+ this.output = output;
+ this.signed = signed;
+ this.alignedBitpacking = alignedBitpacking;
+ this.utils = new SerializationUtils();
+ clear();
+ }
+
+ private void writeValues() throws IOException {
+ if (numLiterals != 0) {
+
+ if (encoding.equals(EncodingType.SHORT_REPEAT)) {
+ writeShortRepeatValues();
+ } else if (encoding.equals(EncodingType.DIRECT)) {
+ writeDirectValues();
+ } else if (encoding.equals(EncodingType.PATCHED_BASE)) {
+ writePatchedBaseValues();
+ } else {
+ writeDeltaValues();
+ }
+
+ // clear all the variables
+ clear();
+ }
+ }
+
+ private void writeDeltaValues() throws IOException {
+ int len = 0;
+ int fb = bitsDeltaMax;
+ int efb = 0;
+
+ if (alignedBitpacking) {
+ fb = utils.getClosestAlignedFixedBits(fb);
+ }
+
+ if (isFixedDelta) {
+ // if fixed run length is greater than threshold then it will be fixed
+ // delta sequence with delta value 0 else fixed delta sequence with
+ // non-zero delta value
+ if (fixedRunLength > MIN_REPEAT) {
+ // ex. sequence: 2 2 2 2 2 2 2 2
+ len = fixedRunLength - 1;
+ fixedRunLength = 0;
+ } else {
+ // ex. sequence: 4 6 8 10 12 14 16
+ len = variableRunLength - 1;
+ variableRunLength = 0;
+ }
+ } else {
+ // fixed width 0 is used for long repeating values.
+ // sequences that require only 1 bit to encode will have an additional bit
+ if (fb == 1) {
+ fb = 2;
+ }
+ efb = utils.encodeBitWidth(fb);
+ efb = efb << 1;
+ len = variableRunLength - 1;
+ variableRunLength = 0;
+ }
+
+ // extract the 9th bit of run length
+ final int tailBits = (len & 0x100) >>> 8;
+
+ // create first byte of the header
+ final int headerFirstByte = getOpcode() | efb | tailBits;
+
+ // second byte of the header stores the remaining 8 bits of runlength
+ final int headerSecondByte = len & 0xff;
+
+ // write header
+ output.write(headerFirstByte);
+ output.write(headerSecondByte);
+
+ // store the first value from zigzag literal array
+ if (signed) {
+ utils.writeVslong(output, literals[0]);
+ } else {
+ utils.writeVulong(output, literals[0]);
+ }
+
+ if (isFixedDelta) {
+ // if delta is fixed then we don't need to store delta blob
+ utils.writeVslong(output, fixedDelta);
+ } else {
+ // store the first value as delta value using zigzag encoding
+ utils.writeVslong(output, adjDeltas[0]);
+
+ // adjacent delta values are bit packed. The length of adjDeltas array is
+ // always one less than the number of literals (delta difference for n
+ // elements is n-1). We have already written one element, write the
+ // remaining numLiterals - 2 elements here
+ utils.writeInts(adjDeltas, 1, numLiterals - 2, fb, output);
+ }
+ }
+
+ private void writePatchedBaseValues() throws IOException {
+
+ // NOTE: Aligned bit packing cannot be applied for PATCHED_BASE encoding
+ // because patch is applied to MSB bits. For example: If fixed bit width of
+ // base value is 7 bits and if patch is 3 bits, the actual value is
+ // constructed by shifting the patch to left by 7 positions.
+ // actual_value = patch << 7 | base_value
+ // So, if we align base_value then actual_value can not be reconstructed.
+
+ // write the number of fixed bits required in next 5 bits
+ final int fb = brBits95p;
+ final int efb = utils.encodeBitWidth(fb) << 1;
+
+ // adjust variable run length, they are one off
+ variableRunLength -= 1;
+
+ // extract the 9th bit of run length
+ final int tailBits = (variableRunLength & 0x100) >>> 8;
+
+ // create first byte of the header
+ final int headerFirstByte = getOpcode() | efb | tailBits;
+
+ // second byte of the header stores the remaining 8 bits of runlength
+ final int headerSecondByte = variableRunLength & 0xff;
+
+ // if the min value is negative toggle the sign
+ final boolean isNegative = min < 0 ? true : false;
+ if (isNegative) {
+ min = -min;
+ }
+
+ // find the number of bytes required for base and shift it by 5 bits
+ // to accommodate patch width. The additional bit is used to store the sign
+ // of the base value.
+ final int baseWidth = utils.findClosestNumBits(min) + 1;
+ final int baseBytes = baseWidth % 8 == 0 ? baseWidth / 8 : (baseWidth / 8) + 1;
+ final int bb = (baseBytes - 1) << 5;
+
+ // if the base value is negative then set MSB to 1
+ if (isNegative) {
+ min |= (1L << ((baseBytes * 8) - 1));
+ }
+
+ // third byte contains 3 bits for number of bytes occupied by base
+ // and 5 bits for patchWidth
+ final int headerThirdByte = bb | utils.encodeBitWidth(patchWidth);
+
+ // fourth byte contains 3 bits for page gap width and 5 bits for
+ // patch length
+ final int headerFourthByte = (patchGapWidth - 1) << 5 | patchLength;
+
+ // write header
+ output.write(headerFirstByte);
+ output.write(headerSecondByte);
+ output.write(headerThirdByte);
+ output.write(headerFourthByte);
+
+ // write the base value using fixed bytes in big endian order
+ for(int i = baseBytes - 1; i >= 0; i--) {
+ byte b = (byte) ((min >>> (i * 8)) & 0xff);
+ output.write(b);
+ }
+
+ // base reduced literals are bit packed
+ int closestFixedBits = utils.getClosestFixedBits(fb);
+
+ utils.writeInts(baseRedLiterals, 0, numLiterals, closestFixedBits,
+ output);
+
+ // write patch list
+ closestFixedBits = utils.getClosestFixedBits(patchGapWidth + patchWidth);
+
+ utils.writeInts(gapVsPatchList, 0, gapVsPatchList.length, closestFixedBits,
+ output);
+
+ // reset run length
+ variableRunLength = 0;
+ }
+
+ /**
+ * Store the opcode in 2 MSB bits
+ * @return opcode
+ */
+ private int getOpcode() {
+ return encoding.ordinal() << 6;
+ }
+
+ private void writeDirectValues() throws IOException {
+
+ // write the number of fixed bits required in next 5 bits
+ int fb = zzBits100p;
+
+ if (alignedBitpacking) {
+ fb = utils.getClosestAlignedFixedBits(fb);
+ }
+
+ final int efb = utils.encodeBitWidth(fb) << 1;
+
+ // adjust variable run length
+ variableRunLength -= 1;
+
+ // extract the 9th bit of run length
+ final int tailBits = (variableRunLength & 0x100) >>> 8;
+
+ // create first byte of the header
+ final int headerFirstByte = getOpcode() | efb | tailBits;
+
+ // second byte of the header stores the remaining 8 bits of runlength
+ final int headerSecondByte = variableRunLength & 0xff;
+
+ // write header
+ output.write(headerFirstByte);
+ output.write(headerSecondByte);
+
+ // bit packing the zigzag encoded literals
+ utils.writeInts(zigzagLiterals, 0, numLiterals, fb, output);
+
+ // reset run length
+ variableRunLength = 0;
+ }
+
+ private void writeShortRepeatValues() throws IOException {
+ // get the value that is repeating, compute the bits and bytes required
+ long repeatVal = 0;
+ if (signed) {
+ repeatVal = utils.zigzagEncode(literals[0]);
+ } else {
+ repeatVal = literals[0];
+ }
+
+ final int numBitsRepeatVal = utils.findClosestNumBits(repeatVal);
+ final int numBytesRepeatVal = numBitsRepeatVal % 8 == 0 ? numBitsRepeatVal >>> 3
+ : (numBitsRepeatVal >>> 3) + 1;
+
+ // write encoding type in top 2 bits
+ int header = getOpcode();
+
+ // write the number of bytes required for the value
+ header |= ((numBytesRepeatVal - 1) << 3);
+
+ // write the run length
+ fixedRunLength -= MIN_REPEAT;
+ header |= fixedRunLength;
+
+ // write the header
+ output.write(header);
+
+ // write the repeating value in big endian byte order
+ for(int i = numBytesRepeatVal - 1; i >= 0; i--) {
+ int b = (int) ((repeatVal >>> (i * 8)) & 0xff);
+ output.write(b);
+ }
+
+ fixedRunLength = 0;
+ }
+
+ private void determineEncoding() {
+
+ // we need to compute zigzag values for DIRECT encoding if we decide to
+ // break early for delta overflows or for shorter runs
+ computeZigZagLiterals();
+
+ zzBits100p = utils.percentileBits(zigzagLiterals, 0, numLiterals, 1.0);
+
+ // not a big win for shorter runs to determine encoding
+ if (numLiterals <= MIN_REPEAT) {
+ encoding = EncodingType.DIRECT;
+ return;
+ }
+
+ // DELTA encoding check
+
+ // for identifying monotonic sequences
+ boolean isIncreasing = true;
+ boolean isDecreasing = true;
+ this.isFixedDelta = true;
+
+ this.min = literals[0];
+ long max = literals[0];
+ final long initialDelta = literals[1] - literals[0];
+ long currDelta = initialDelta;
+ long deltaMax = initialDelta;
+ this.adjDeltas[0] = initialDelta;
+
+ for (int i = 1; i < numLiterals; i++) {
+ final long l1 = literals[i];
+ final long l0 = literals[i - 1];
+ currDelta = l1 - l0;
+ min = Math.min(min, l1);
+ max = Math.max(max, l1);
+
+ isIncreasing &= (l0 <= l1);
+ isDecreasing &= (l0 >= l1);
+
+ isFixedDelta &= (currDelta == initialDelta);
+ if (i > 1) {
+ adjDeltas[i - 1] = Math.abs(currDelta);
+ deltaMax = Math.max(deltaMax, adjDeltas[i - 1]);
+ }
+ }
+
+ // its faster to exit under delta overflow condition without checking for
+ // PATCHED_BASE condition as encoding using DIRECT is faster and has less
+ // overhead than PATCHED_BASE
+ if (!utils.isSafeSubtract(max, min)) {
+ encoding = EncodingType.DIRECT;
+ return;
+ }
+
+ // invariant - subtracting any number from any other in the literals after
+ // this point won't overflow
+
+ // if min is equal to max then the delta is 0, this condition happens for
+ // fixed values run >10 which cannot be encoded with SHORT_REPEAT
+ if (min == max) {
+ assert isFixedDelta : min + "==" + max +
+ ", isFixedDelta cannot be false";
+ assert currDelta == 0 : min + "==" + max + ", currDelta should be zero";
+ fixedDelta = 0;
+ encoding = EncodingType.DELTA;
+ return;
+ }
+
+ if (isFixedDelta) {
+ assert currDelta == initialDelta
+ : "currDelta should be equal to initialDelta for fixed delta encoding";
+ encoding = EncodingType.DELTA;
+ fixedDelta = currDelta;
+ return;
+ }
+
+ // if initialDelta is 0 then we cannot delta encode as we cannot identify
+ // the sign of deltas (increasing or decreasing)
+ if (initialDelta != 0) {
+ // stores the number of bits required for packing delta blob in
+ // delta encoding
+ bitsDeltaMax = utils.findClosestNumBits(deltaMax);
+
+ // monotonic condition
+ if (isIncreasing || isDecreasing) {
+ encoding = EncodingType.DELTA;
+ return;
+ }
+ }
+
+ // PATCHED_BASE encoding check
+
+ // percentile values are computed for the zigzag encoded values. if the
+ // number of bit requirement between 90th and 100th percentile varies
+ // beyond a threshold then we need to patch the values. if the variation
+ // is not significant then we can use direct encoding
+
+ zzBits90p = utils.percentileBits(zigzagLiterals, 0, numLiterals, 0.9);
+ int diffBitsLH = zzBits100p - zzBits90p;
+
+ // if the difference between 90th percentile and 100th percentile fixed
+ // bits is > 1 then we need patch the values
+ if (diffBitsLH > 1) {
+
+ // patching is done only on base reduced values.
+ // remove base from literals
+ for (int i = 0; i < numLiterals; i++) {
+ baseRedLiterals[i] = literals[i] - min;
+ }
+
+ // 95th percentile width is used to determine max allowed value
+ // after which patching will be done
+ brBits95p = utils.percentileBits(baseRedLiterals, 0, numLiterals, 0.95);
+
+ // 100th percentile is used to compute the max patch width
+ brBits100p = utils.percentileBits(baseRedLiterals, 0, numLiterals, 1.0);
+
+ // after base reducing the values, if the difference in bits between
+ // 95th percentile and 100th percentile value is zero then there
+ // is no point in patching the values, in which case we will
+ // fallback to DIRECT encoding.
+ // The decision to use patched base was based on zigzag values, but the
+ // actual patching is done on base reduced literals.
+ if ((brBits100p - brBits95p) != 0) {
+ encoding = EncodingType.PATCHED_BASE;
+ preparePatchedBlob();
+ return;
+ } else {
+ encoding = EncodingType.DIRECT;
+ return;
+ }
+ } else {
+ // if difference in bits between 95th percentile and 100th percentile is
+ // 0, then patch length will become 0. Hence we will fallback to direct
+ encoding = EncodingType.DIRECT;
+ return;
+ }
+ }
+
+ private void computeZigZagLiterals() {
+ // populate zigzag encoded literals
+ long zzEncVal = 0;
+ for (int i = 0; i < numLiterals; i++) {
+ if (signed) {
+ zzEncVal = utils.zigzagEncode(literals[i]);
+ } else {
+ zzEncVal = literals[i];
+ }
+ zigzagLiterals[i] = zzEncVal;
+ }
+ }
+
+ private void preparePatchedBlob() {
+ // mask will be max value beyond which patch will be generated
+ long mask = (1L << brBits95p) - 1;
+
+ // since we are considering only 95 percentile, the size of gap and
+ // patch array can contain only be 5% values
+ patchLength = (int) Math.ceil((numLiterals * 0.05));
+
+ int[] gapList = new int[patchLength];
+ long[] patchList = new long[patchLength];
+
+ // #bit for patch
+ patchWidth = brBits100p - brBits95p;
+ patchWidth = utils.getClosestFixedBits(patchWidth);
+
+ // if patch bit requirement is 64 then it will not possible to pack
+ // gap and patch together in a long. To make sure gap and patch can be
+ // packed together adjust the patch width
+ if (patchWidth == 64) {
+ patchWidth = 56;
+ brBits95p = 8;
+ mask = (1L << brBits95p) - 1;
+ }
+
+ int gapIdx = 0;
+ int patchIdx = 0;
+ int prev = 0;
+ int gap = 0;
+ int maxGap = 0;
+
+ for(int i = 0; i < numLiterals; i++) {
+ // if value is above mask then create the patch and record the gap
+ if (baseRedLiterals[i] > mask) {
+ gap = i - prev;
+ if (gap > maxGap) {
+ maxGap = gap;
+ }
+
+ // gaps are relative, so store the previous patched value index
+ prev = i;
+ gapList[gapIdx++] = gap;
+
+ // extract the most significant bits that are over mask bits
+ long patch = baseRedLiterals[i] >>> brBits95p;
+ patchList[patchIdx++] = patch;
+
+ // strip off the MSB to enable safe bit packing
+ baseRedLiterals[i] &= mask;
+ }
+ }
+
+ // adjust the patch length to number of entries in gap list
+ patchLength = gapIdx;
+
+ // if the element to be patched is the first and only element then
+ // max gap will be 0, but to store the gap as 0 we need atleast 1 bit
+ if (maxGap == 0 && patchLength != 0) {
+ patchGapWidth = 1;
+ } else {
+ patchGapWidth = utils.findClosestNumBits(maxGap);
+ }
+
+ // special case: if the patch gap width is greater than 256, then
+ // we need 9 bits to encode the gap width. But we only have 3 bits in
+ // header to record the gap width. To deal with this case, we will save
+ // two entries in patch list in the following way
+ // 256 gap width => 0 for patch value
+ // actual gap - 256 => actual patch value
+ // We will do the same for gap width = 511. If the element to be patched is
+ // the last element in the scope then gap width will be 511. In this case we
+ // will have 3 entries in the patch list in the following way
+ // 255 gap width => 0 for patch value
+ // 255 gap width => 0 for patch value
+ // 1 gap width => actual patch value
+ if (patchGapWidth > 8) {
+ patchGapWidth = 8;
+ // for gap = 511, we need two additional entries in patch list
+ if (maxGap == 511) {
+ patchLength += 2;
+ } else {
+ patchLength += 1;
+ }
+ }
+
+ // create gap vs patch list
+ gapIdx = 0;
+ patchIdx = 0;
+ gapVsPatchList = new long[patchLength];
+ for(int i = 0; i < patchLength; i++) {
+ long g = gapList[gapIdx++];
+ long p = patchList[patchIdx++];
+ while (g > 255) {
+ gapVsPatchList[i++] = (255L << patchWidth);
+ g -= 255;
+ }
+
+ // store patch value in LSBs and gap in MSBs
+ gapVsPatchList[i] = (g << patchWidth) | p;
+ }
+ }
+
+ /**
+ * clears all the variables
+ */
+ private void clear() {
+ numLiterals = 0;
+ encoding = null;
+ prevDelta = 0;
+ fixedDelta = 0;
+ zzBits90p = 0;
+ zzBits100p = 0;
+ brBits95p = 0;
+ brBits100p = 0;
+ bitsDeltaMax = 0;
+ patchGapWidth = 0;
+ patchLength = 0;
+ patchWidth = 0;
+ gapVsPatchList = null;
+ min = 0;
+ isFixedDelta = true;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ if (numLiterals != 0) {
+ if (variableRunLength != 0) {
+ determineEncoding();
+ writeValues();
+ } else if (fixedRunLength != 0) {
+ if (fixedRunLength < MIN_REPEAT) {
+ variableRunLength = fixedRunLength;
+ fixedRunLength = 0;
+ determineEncoding();
+ writeValues();
+ } else if (fixedRunLength >= MIN_REPEAT
+ && fixedRunLength <= MAX_SHORT_REPEAT_LENGTH) {
+ encoding = EncodingType.SHORT_REPEAT;
+ writeValues();
+ } else {
+ encoding = EncodingType.DELTA;
+ isFixedDelta = true;
+ writeValues();
+ }
+ }
+ }
+ output.flush();
+ }
+
+ @Override
+ public void write(long val) throws IOException {
+ if (numLiterals == 0) {
+ initializeLiterals(val);
+ } else {
+ if (numLiterals == 1) {
+ prevDelta = val - literals[0];
+ literals[numLiterals++] = val;
+ // if both values are same count as fixed run else variable run
+ if (val == literals[0]) {
+ fixedRunLength = 2;
+ variableRunLength = 0;
+ } else {
+ fixedRunLength = 0;
+ variableRunLength = 2;
+ }
+ } else {
+ long currentDelta = val - literals[numLiterals - 1];
+ if (prevDelta == 0 && currentDelta == 0) {
+ // fixed delta run
+
+ literals[numLiterals++] = val;
+
+ // if variable run is non-zero then we are seeing repeating
+ // values at the end of variable run in which case keep
+ // updating variable and fixed runs
+ if (variableRunLength > 0) {
+ fixedRunLength = 2;
+ }
+ fixedRunLength += 1;
+
+ // if fixed run met the minimum condition and if variable
+ // run is non-zero then flush the variable run and shift the
+ // tail fixed runs to start of the buffer
+ if (fixedRunLength >= MIN_REPEAT && variableRunLength > 0) {
+ numLiterals -= MIN_REPEAT;
+ variableRunLength -= MIN_REPEAT - 1;
+ // copy the tail fixed runs
+ long[] tailVals = new long[MIN_REPEAT];
+ System.arraycopy(literals, numLiterals, tailVals, 0, MIN_REPEAT);
+
+ // determine variable encoding and flush values
+ determineEncoding();
+ writeValues();
+
+ // shift tail fixed runs to beginning of the buffer
+ for(long l : tailVals) {
+ literals[numLiterals++] = l;
+ }
+ }
+
+ // if fixed runs reached max repeat length then write values
+ if (fixedRunLength == MAX_SCOPE) {
+ determineEncoding();
+ writeValues();
+ }
+ } else {
+ // variable delta run
+
+ // if fixed run length is non-zero and if it satisfies the
+ // short repeat conditions then write the values as short repeats
+ // else use delta encoding
+ if (fixedRunLength >= MIN_REPEAT) {
+ if (fixedRunLength <= MAX_SHORT_REPEAT_LENGTH) {
+ encoding = EncodingType.SHORT_REPEAT;
+ writeValues();
+ } else {
+ encoding = EncodingType.DELTA;
+ isFixedDelta = true;
+ writeValues();
+ }
+ }
+
+ // if fixed run length is <MIN_REPEAT and current value is
+ // different from previous then treat it as variable run
+ if (fixedRunLength > 0 && fixedRunLength < MIN_REPEAT) {
+ if (val != literals[numLiterals - 1]) {
+ variableRunLength = fixedRunLength;
+ fixedRunLength = 0;
+ }
+ }
+
+ // after writing values re-initialize the variables
+ if (numLiterals == 0) {
+ initializeLiterals(val);
+ } else {
+ // keep updating variable run lengths
+ prevDelta = val - literals[numLiterals - 1];
+ literals[numLiterals++] = val;
+ variableRunLength += 1;
+
+ // if variable run length reach the max scope, write it
+ if (variableRunLength == MAX_SCOPE) {
+ determineEncoding();
+ writeValues();
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private void initializeLiterals(long val) {
+ literals[numLiterals++] = val;
+ fixedRunLength = 1;
+ variableRunLength = 1;
+ }
+
+ @Override
+ public void getPosition(PositionRecorder recorder) throws IOException {
+ output.getPosition(recorder);
+ recorder.addPosition(numLiterals);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/SerializationUtils.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/SerializationUtils.java b/orc/src/java/org/apache/orc/impl/SerializationUtils.java
new file mode 100644
index 0000000..c1162e4
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/SerializationUtils.java
@@ -0,0 +1,1297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.orc.impl.InStream;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigInteger;
+
+public final class SerializationUtils {
+
+ private final static int BUFFER_SIZE = 64;
+ private final byte[] readBuffer;
+ private final byte[] writeBuffer;
+
+ public SerializationUtils() {
+ this.readBuffer = new byte[BUFFER_SIZE];
+ this.writeBuffer = new byte[BUFFER_SIZE];
+ }
+
+ public void writeVulong(OutputStream output,
+ long value) throws IOException {
+ while (true) {
+ if ((value & ~0x7f) == 0) {
+ output.write((byte) value);
+ return;
+ } else {
+ output.write((byte) (0x80 | (value & 0x7f)));
+ value >>>= 7;
+ }
+ }
+ }
+
+ public void writeVslong(OutputStream output,
+ long value) throws IOException {
+ writeVulong(output, (value << 1) ^ (value >> 63));
+ }
+
+
+ public long readVulong(InputStream in) throws IOException {
+ long result = 0;
+ long b;
+ int offset = 0;
+ do {
+ b = in.read();
+ if (b == -1) {
+ throw new EOFException("Reading Vulong past EOF");
+ }
+ result |= (0x7f & b) << offset;
+ offset += 7;
+ } while (b >= 0x80);
+ return result;
+ }
+
+ public long readVslong(InputStream in) throws IOException {
+ long result = readVulong(in);
+ return (result >>> 1) ^ -(result & 1);
+ }
+
+ public float readFloat(InputStream in) throws IOException {
+ int ser = in.read() | (in.read() << 8) | (in.read() << 16) |
+ (in.read() << 24);
+ return Float.intBitsToFloat(ser);
+ }
+
+ public void writeFloat(OutputStream output,
+ float value) throws IOException {
+ int ser = Float.floatToIntBits(value);
+ output.write(ser & 0xff);
+ output.write((ser >> 8) & 0xff);
+ output.write((ser >> 16) & 0xff);
+ output.write((ser >> 24) & 0xff);
+ }
+
+ public double readDouble(InputStream in) throws IOException {
+ return Double.longBitsToDouble(readLongLE(in));
+ }
+
+ public long readLongLE(InputStream in) throws IOException {
+ in.read(readBuffer, 0, 8);
+ return (((readBuffer[0] & 0xff) << 0)
+ + ((readBuffer[1] & 0xff) << 8)
+ + ((readBuffer[2] & 0xff) << 16)
+ + ((long) (readBuffer[3] & 0xff) << 24)
+ + ((long) (readBuffer[4] & 0xff) << 32)
+ + ((long) (readBuffer[5] & 0xff) << 40)
+ + ((long) (readBuffer[6] & 0xff) << 48)
+ + ((long) (readBuffer[7] & 0xff) << 56));
+ }
+
+ public void writeDouble(OutputStream output,
+ double value) throws IOException {
+ writeLongLE(output, Double.doubleToLongBits(value));
+ }
+
+ private void writeLongLE(OutputStream output, long value) throws IOException {
+ writeBuffer[0] = (byte) ((value >> 0) & 0xff);
+ writeBuffer[1] = (byte) ((value >> 8) & 0xff);
+ writeBuffer[2] = (byte) ((value >> 16) & 0xff);
+ writeBuffer[3] = (byte) ((value >> 24) & 0xff);
+ writeBuffer[4] = (byte) ((value >> 32) & 0xff);
+ writeBuffer[5] = (byte) ((value >> 40) & 0xff);
+ writeBuffer[6] = (byte) ((value >> 48) & 0xff);
+ writeBuffer[7] = (byte) ((value >> 56) & 0xff);
+ output.write(writeBuffer, 0, 8);
+ }
+
+ /**
+ * Write the arbitrarily sized signed BigInteger in vint format.
+ *
+ * Signed integers are encoded using the low bit as the sign bit using zigzag
+ * encoding.
+ *
+ * Each byte uses the low 7 bits for data and the high bit for stop/continue.
+ *
+ * Bytes are stored LSB first.
+ * @param output the stream to write to
+ * @param value the value to output
+ * @throws IOException
+ */
+ public static void writeBigInteger(OutputStream output,
+ BigInteger value) throws IOException {
+ // encode the signed number as a positive integer
+ value = value.shiftLeft(1);
+ int sign = value.signum();
+ if (sign < 0) {
+ value = value.negate();
+ value = value.subtract(BigInteger.ONE);
+ }
+ int length = value.bitLength();
+ while (true) {
+ long lowBits = value.longValue() & 0x7fffffffffffffffL;
+ length -= 63;
+ // write out the next 63 bits worth of data
+ for(int i=0; i < 9; ++i) {
+ // if this is the last byte, leave the high bit off
+ if (length <= 0 && (lowBits & ~0x7f) == 0) {
+ output.write((byte) lowBits);
+ return;
+ } else {
+ output.write((byte) (0x80 | (lowBits & 0x7f)));
+ lowBits >>>= 7;
+ }
+ }
+ value = value.shiftRight(63);
+ }
+ }
+
+ /**
+ * Read the signed arbitrary sized BigInteger BigInteger in vint format
+ * @param input the stream to read from
+ * @return the read BigInteger
+ * @throws IOException
+ */
+ public static BigInteger readBigInteger(InputStream input) throws IOException {
+ BigInteger result = BigInteger.ZERO;
+ long work = 0;
+ int offset = 0;
+ long b;
+ do {
+ b = input.read();
+ if (b == -1) {
+ throw new EOFException("Reading BigInteger past EOF from " + input);
+ }
+ work |= (0x7f & b) << (offset % 63);
+ offset += 7;
+ // if we've read 63 bits, roll them into the result
+ if (offset == 63) {
+ result = BigInteger.valueOf(work);
+ work = 0;
+ } else if (offset % 63 == 0) {
+ result = result.or(BigInteger.valueOf(work).shiftLeft(offset-63));
+ work = 0;
+ }
+ } while (b >= 0x80);
+ if (work != 0) {
+ result = result.or(BigInteger.valueOf(work).shiftLeft((offset/63)*63));
+ }
+ // convert back to a signed number
+ boolean isNegative = result.testBit(0);
+ if (isNegative) {
+ result = result.add(BigInteger.ONE);
+ result = result.negate();
+ }
+ result = result.shiftRight(1);
+ return result;
+ }
+
+ public enum FixedBitSizes {
+ ONE, TWO, THREE, FOUR, FIVE, SIX, SEVEN, EIGHT, NINE, TEN, ELEVEN, TWELVE,
+ THIRTEEN, FOURTEEN, FIFTEEN, SIXTEEN, SEVENTEEN, EIGHTEEN, NINETEEN,
+ TWENTY, TWENTYONE, TWENTYTWO, TWENTYTHREE, TWENTYFOUR, TWENTYSIX,
+ TWENTYEIGHT, THIRTY, THIRTYTWO, FORTY, FORTYEIGHT, FIFTYSIX, SIXTYFOUR;
+ }
+
+ /**
+ * Count the number of bits required to encode the given value
+ * @param value
+ * @return bits required to store value
+ */
+ public int findClosestNumBits(long value) {
+ int count = 0;
+ while (value != 0) {
+ count++;
+ value = value >>> 1;
+ }
+ return getClosestFixedBits(count);
+ }
+
+ /**
+ * zigzag encode the given value
+ * @param val
+ * @return zigzag encoded value
+ */
+ public long zigzagEncode(long val) {
+ return (val << 1) ^ (val >> 63);
+ }
+
+ /**
+ * zigzag decode the given value
+ * @param val
+ * @return zizag decoded value
+ */
+ public long zigzagDecode(long val) {
+ return (val >>> 1) ^ -(val & 1);
+ }
+
+ /**
+ * Compute the bits required to represent pth percentile value
+ * @param data - array
+ * @param p - percentile value (>=0.0 to <=1.0)
+ * @return pth percentile bits
+ */
+ public int percentileBits(long[] data, int offset, int length,
+ double p) {
+ if ((p > 1.0) || (p <= 0.0)) {
+ return -1;
+ }
+
+ // histogram that store the encoded bit requirement for each values.
+ // maximum number of bits that can encoded is 32 (refer FixedBitSizes)
+ int[] hist = new int[32];
+
+ // compute the histogram
+ for(int i = offset; i < (offset + length); i++) {
+ int idx = encodeBitWidth(findClosestNumBits(data[i]));
+ hist[idx] += 1;
+ }
+
+ int perLen = (int) (length * (1.0 - p));
+
+ // return the bits required by pth percentile length
+ for(int i = hist.length - 1; i >= 0; i--) {
+ perLen -= hist[i];
+ if (perLen < 0) {
+ return decodeBitWidth(i);
+ }
+ }
+
+ return 0;
+ }
+
+ /**
+ * Read n bytes in big endian order and convert to long
+ * @return long value
+ */
+ public long bytesToLongBE(InStream input, int n) throws IOException {
+ long out = 0;
+ long val = 0;
+ while (n > 0) {
+ n--;
+ // store it in a long and then shift else integer overflow will occur
+ val = input.read();
+ out |= (val << (n * 8));
+ }
+ return out;
+ }
+
+ /**
+ * Calculate the number of bytes required
+ * @param n - number of values
+ * @param numBits - bit width
+ * @return number of bytes required
+ */
+ int getTotalBytesRequired(int n, int numBits) {
+ return (n * numBits + 7) / 8;
+ }
+
+ /**
+ * For a given fixed bit this function will return the closest available fixed
+ * bit
+ * @param n
+ * @return closest valid fixed bit
+ */
+ public int getClosestFixedBits(int n) {
+ if (n == 0) {
+ return 1;
+ }
+
+ if (n >= 1 && n <= 24) {
+ return n;
+ } else if (n > 24 && n <= 26) {
+ return 26;
+ } else if (n > 26 && n <= 28) {
+ return 28;
+ } else if (n > 28 && n <= 30) {
+ return 30;
+ } else if (n > 30 && n <= 32) {
+ return 32;
+ } else if (n > 32 && n <= 40) {
+ return 40;
+ } else if (n > 40 && n <= 48) {
+ return 48;
+ } else if (n > 48 && n <= 56) {
+ return 56;
+ } else {
+ return 64;
+ }
+ }
+
+ public int getClosestAlignedFixedBits(int n) {
+ if (n == 0 || n == 1) {
+ return 1;
+ } else if (n > 1 && n <= 2) {
+ return 2;
+ } else if (n > 2 && n <= 4) {
+ return 4;
+ } else if (n > 4 && n <= 8) {
+ return 8;
+ } else if (n > 8 && n <= 16) {
+ return 16;
+ } else if (n > 16 && n <= 24) {
+ return 24;
+ } else if (n > 24 && n <= 32) {
+ return 32;
+ } else if (n > 32 && n <= 40) {
+ return 40;
+ } else if (n > 40 && n <= 48) {
+ return 48;
+ } else if (n > 48 && n <= 56) {
+ return 56;
+ } else {
+ return 64;
+ }
+ }
+
+ /**
+ * Finds the closest available fixed bit width match and returns its encoded
+ * value (ordinal)
+ * @param n - fixed bit width to encode
+ * @return encoded fixed bit width
+ */
+ public int encodeBitWidth(int n) {
+ n = getClosestFixedBits(n);
+
+ if (n >= 1 && n <= 24) {
+ return n - 1;
+ } else if (n > 24 && n <= 26) {
+ return FixedBitSizes.TWENTYSIX.ordinal();
+ } else if (n > 26 && n <= 28) {
+ return FixedBitSizes.TWENTYEIGHT.ordinal();
+ } else if (n > 28 && n <= 30) {
+ return FixedBitSizes.THIRTY.ordinal();
+ } else if (n > 30 && n <= 32) {
+ return FixedBitSizes.THIRTYTWO.ordinal();
+ } else if (n > 32 && n <= 40) {
+ return FixedBitSizes.FORTY.ordinal();
+ } else if (n > 40 && n <= 48) {
+ return FixedBitSizes.FORTYEIGHT.ordinal();
+ } else if (n > 48 && n <= 56) {
+ return FixedBitSizes.FIFTYSIX.ordinal();
+ } else {
+ return FixedBitSizes.SIXTYFOUR.ordinal();
+ }
+ }
+
+ /**
+ * Decodes the ordinal fixed bit value to actual fixed bit width value
+ * @param n - encoded fixed bit width
+ * @return decoded fixed bit width
+ */
+ public int decodeBitWidth(int n) {
+ if (n >= FixedBitSizes.ONE.ordinal()
+ && n <= FixedBitSizes.TWENTYFOUR.ordinal()) {
+ return n + 1;
+ } else if (n == FixedBitSizes.TWENTYSIX.ordinal()) {
+ return 26;
+ } else if (n == FixedBitSizes.TWENTYEIGHT.ordinal()) {
+ return 28;
+ } else if (n == FixedBitSizes.THIRTY.ordinal()) {
+ return 30;
+ } else if (n == FixedBitSizes.THIRTYTWO.ordinal()) {
+ return 32;
+ } else if (n == FixedBitSizes.FORTY.ordinal()) {
+ return 40;
+ } else if (n == FixedBitSizes.FORTYEIGHT.ordinal()) {
+ return 48;
+ } else if (n == FixedBitSizes.FIFTYSIX.ordinal()) {
+ return 56;
+ } else {
+ return 64;
+ }
+ }
+
+ /**
+ * Bitpack and write the input values to underlying output stream
+ * @param input - values to write
+ * @param offset - offset
+ * @param len - length
+ * @param bitSize - bit width
+ * @param output - output stream
+ * @throws IOException
+ */
+ public void writeInts(long[] input, int offset, int len, int bitSize,
+ OutputStream output) throws IOException {
+ if (input == null || input.length < 1 || offset < 0 || len < 1
+ || bitSize < 1) {
+ return;
+ }
+
+ switch (bitSize) {
+ case 1:
+ unrolledBitPack1(input, offset, len, output);
+ return;
+ case 2:
+ unrolledBitPack2(input, offset, len, output);
+ return;
+ case 4:
+ unrolledBitPack4(input, offset, len, output);
+ return;
+ case 8:
+ unrolledBitPack8(input, offset, len, output);
+ return;
+ case 16:
+ unrolledBitPack16(input, offset, len, output);
+ return;
+ case 24:
+ unrolledBitPack24(input, offset, len, output);
+ return;
+ case 32:
+ unrolledBitPack32(input, offset, len, output);
+ return;
+ case 40:
+ unrolledBitPack40(input, offset, len, output);
+ return;
+ case 48:
+ unrolledBitPack48(input, offset, len, output);
+ return;
+ case 56:
+ unrolledBitPack56(input, offset, len, output);
+ return;
+ case 64:
+ unrolledBitPack64(input, offset, len, output);
+ return;
+ default:
+ break;
+ }
+
+ int bitsLeft = 8;
+ byte current = 0;
+ for(int i = offset; i < (offset + len); i++) {
+ long value = input[i];
+ int bitsToWrite = bitSize;
+ while (bitsToWrite > bitsLeft) {
+ // add the bits to the bottom of the current word
+ current |= value >>> (bitsToWrite - bitsLeft);
+ // subtract out the bits we just added
+ bitsToWrite -= bitsLeft;
+ // zero out the bits above bitsToWrite
+ value &= (1L << bitsToWrite) - 1;
+ output.write(current);
+ current = 0;
+ bitsLeft = 8;
+ }
+ bitsLeft -= bitsToWrite;
+ current |= value << bitsLeft;
+ if (bitsLeft == 0) {
+ output.write(current);
+ current = 0;
+ bitsLeft = 8;
+ }
+ }
+
+ // flush
+ if (bitsLeft != 8) {
+ output.write(current);
+ current = 0;
+ bitsLeft = 8;
+ }
+ }
+
+ private void unrolledBitPack1(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ final int numHops = 8;
+ final int remainder = len % numHops;
+ final int endOffset = offset + len;
+ final int endUnroll = endOffset - remainder;
+ int val = 0;
+ for (int i = offset; i < endUnroll; i = i + numHops) {
+ val = (int) (val | ((input[i] & 1) << 7)
+ | ((input[i + 1] & 1) << 6)
+ | ((input[i + 2] & 1) << 5)
+ | ((input[i + 3] & 1) << 4)
+ | ((input[i + 4] & 1) << 3)
+ | ((input[i + 5] & 1) << 2)
+ | ((input[i + 6] & 1) << 1)
+ | (input[i + 7]) & 1);
+ output.write(val);
+ val = 0;
+ }
+
+ if (remainder > 0) {
+ int startShift = 7;
+ for (int i = endUnroll; i < endOffset; i++) {
+ val = (int) (val | (input[i] & 1) << startShift);
+ startShift -= 1;
+ }
+ output.write(val);
+ }
+ }
+
+ private void unrolledBitPack2(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ final int numHops = 4;
+ final int remainder = len % numHops;
+ final int endOffset = offset + len;
+ final int endUnroll = endOffset - remainder;
+ int val = 0;
+ for (int i = offset; i < endUnroll; i = i + numHops) {
+ val = (int) (val | ((input[i] & 3) << 6)
+ | ((input[i + 1] & 3) << 4)
+ | ((input[i + 2] & 3) << 2)
+ | (input[i + 3]) & 3);
+ output.write(val);
+ val = 0;
+ }
+
+ if (remainder > 0) {
+ int startShift = 6;
+ for (int i = endUnroll; i < endOffset; i++) {
+ val = (int) (val | (input[i] & 3) << startShift);
+ startShift -= 2;
+ }
+ output.write(val);
+ }
+ }
+
+ private void unrolledBitPack4(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ final int numHops = 2;
+ final int remainder = len % numHops;
+ final int endOffset = offset + len;
+ final int endUnroll = endOffset - remainder;
+ int val = 0;
+ for (int i = offset; i < endUnroll; i = i + numHops) {
+ val = (int) (val | ((input[i] & 15) << 4) | (input[i + 1]) & 15);
+ output.write(val);
+ val = 0;
+ }
+
+ if (remainder > 0) {
+ int startShift = 4;
+ for (int i = endUnroll; i < endOffset; i++) {
+ val = (int) (val | (input[i] & 15) << startShift);
+ startShift -= 4;
+ }
+ output.write(val);
+ }
+ }
+
+ private void unrolledBitPack8(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ unrolledBitPackBytes(input, offset, len, output, 1);
+ }
+
+ private void unrolledBitPack16(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ unrolledBitPackBytes(input, offset, len, output, 2);
+ }
+
+ private void unrolledBitPack24(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ unrolledBitPackBytes(input, offset, len, output, 3);
+ }
+
+ private void unrolledBitPack32(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ unrolledBitPackBytes(input, offset, len, output, 4);
+ }
+
+ private void unrolledBitPack40(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ unrolledBitPackBytes(input, offset, len, output, 5);
+ }
+
+ private void unrolledBitPack48(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ unrolledBitPackBytes(input, offset, len, output, 6);
+ }
+
+ private void unrolledBitPack56(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ unrolledBitPackBytes(input, offset, len, output, 7);
+ }
+
+ private void unrolledBitPack64(long[] input, int offset, int len,
+ OutputStream output) throws IOException {
+ unrolledBitPackBytes(input, offset, len, output, 8);
+ }
+
+ private void unrolledBitPackBytes(long[] input, int offset, int len, OutputStream output, int numBytes) throws IOException {
+ final int numHops = 8;
+ final int remainder = len % numHops;
+ final int endOffset = offset + len;
+ final int endUnroll = endOffset - remainder;
+ int i = offset;
+ for (; i < endUnroll; i = i + numHops) {
+ writeLongBE(output, input, i, numHops, numBytes);
+ }
+
+ if (remainder > 0) {
+ writeRemainingLongs(output, i, input, remainder, numBytes);
+ }
+ }
+
+ private void writeRemainingLongs(OutputStream output, int offset, long[] input, int remainder,
+ int numBytes) throws IOException {
+ final int numHops = remainder;
+
+ int idx = 0;
+ switch (numBytes) {
+ case 1:
+ while (remainder > 0) {
+ writeBuffer[idx] = (byte) (input[offset + idx] & 255);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 2:
+ while (remainder > 0) {
+ writeLongBE2(output, input[offset + idx], idx * 2);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 3:
+ while (remainder > 0) {
+ writeLongBE3(output, input[offset + idx], idx * 3);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 4:
+ while (remainder > 0) {
+ writeLongBE4(output, input[offset + idx], idx * 4);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 5:
+ while (remainder > 0) {
+ writeLongBE5(output, input[offset + idx], idx * 5);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 6:
+ while (remainder > 0) {
+ writeLongBE6(output, input[offset + idx], idx * 6);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 7:
+ while (remainder > 0) {
+ writeLongBE7(output, input[offset + idx], idx * 7);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 8:
+ while (remainder > 0) {
+ writeLongBE8(output, input[offset + idx], idx * 8);
+ remainder--;
+ idx++;
+ }
+ break;
+ default:
+ break;
+ }
+
+ final int toWrite = numHops * numBytes;
+ output.write(writeBuffer, 0, toWrite);
+ }
+
+ private void writeLongBE(OutputStream output, long[] input, int offset, int numHops, int numBytes) throws IOException {
+
+ switch (numBytes) {
+ case 1:
+ writeBuffer[0] = (byte) (input[offset + 0] & 255);
+ writeBuffer[1] = (byte) (input[offset + 1] & 255);
+ writeBuffer[2] = (byte) (input[offset + 2] & 255);
+ writeBuffer[3] = (byte) (input[offset + 3] & 255);
+ writeBuffer[4] = (byte) (input[offset + 4] & 255);
+ writeBuffer[5] = (byte) (input[offset + 5] & 255);
+ writeBuffer[6] = (byte) (input[offset + 6] & 255);
+ writeBuffer[7] = (byte) (input[offset + 7] & 255);
+ break;
+ case 2:
+ writeLongBE2(output, input[offset + 0], 0);
+ writeLongBE2(output, input[offset + 1], 2);
+ writeLongBE2(output, input[offset + 2], 4);
+ writeLongBE2(output, input[offset + 3], 6);
+ writeLongBE2(output, input[offset + 4], 8);
+ writeLongBE2(output, input[offset + 5], 10);
+ writeLongBE2(output, input[offset + 6], 12);
+ writeLongBE2(output, input[offset + 7], 14);
+ break;
+ case 3:
+ writeLongBE3(output, input[offset + 0], 0);
+ writeLongBE3(output, input[offset + 1], 3);
+ writeLongBE3(output, input[offset + 2], 6);
+ writeLongBE3(output, input[offset + 3], 9);
+ writeLongBE3(output, input[offset + 4], 12);
+ writeLongBE3(output, input[offset + 5], 15);
+ writeLongBE3(output, input[offset + 6], 18);
+ writeLongBE3(output, input[offset + 7], 21);
+ break;
+ case 4:
+ writeLongBE4(output, input[offset + 0], 0);
+ writeLongBE4(output, input[offset + 1], 4);
+ writeLongBE4(output, input[offset + 2], 8);
+ writeLongBE4(output, input[offset + 3], 12);
+ writeLongBE4(output, input[offset + 4], 16);
+ writeLongBE4(output, input[offset + 5], 20);
+ writeLongBE4(output, input[offset + 6], 24);
+ writeLongBE4(output, input[offset + 7], 28);
+ break;
+ case 5:
+ writeLongBE5(output, input[offset + 0], 0);
+ writeLongBE5(output, input[offset + 1], 5);
+ writeLongBE5(output, input[offset + 2], 10);
+ writeLongBE5(output, input[offset + 3], 15);
+ writeLongBE5(output, input[offset + 4], 20);
+ writeLongBE5(output, input[offset + 5], 25);
+ writeLongBE5(output, input[offset + 6], 30);
+ writeLongBE5(output, input[offset + 7], 35);
+ break;
+ case 6:
+ writeLongBE6(output, input[offset + 0], 0);
+ writeLongBE6(output, input[offset + 1], 6);
+ writeLongBE6(output, input[offset + 2], 12);
+ writeLongBE6(output, input[offset + 3], 18);
+ writeLongBE6(output, input[offset + 4], 24);
+ writeLongBE6(output, input[offset + 5], 30);
+ writeLongBE6(output, input[offset + 6], 36);
+ writeLongBE6(output, input[offset + 7], 42);
+ break;
+ case 7:
+ writeLongBE7(output, input[offset + 0], 0);
+ writeLongBE7(output, input[offset + 1], 7);
+ writeLongBE7(output, input[offset + 2], 14);
+ writeLongBE7(output, input[offset + 3], 21);
+ writeLongBE7(output, input[offset + 4], 28);
+ writeLongBE7(output, input[offset + 5], 35);
+ writeLongBE7(output, input[offset + 6], 42);
+ writeLongBE7(output, input[offset + 7], 49);
+ break;
+ case 8:
+ writeLongBE8(output, input[offset + 0], 0);
+ writeLongBE8(output, input[offset + 1], 8);
+ writeLongBE8(output, input[offset + 2], 16);
+ writeLongBE8(output, input[offset + 3], 24);
+ writeLongBE8(output, input[offset + 4], 32);
+ writeLongBE8(output, input[offset + 5], 40);
+ writeLongBE8(output, input[offset + 6], 48);
+ writeLongBE8(output, input[offset + 7], 56);
+ break;
+ default:
+ break;
+ }
+
+ final int toWrite = numHops * numBytes;
+ output.write(writeBuffer, 0, toWrite);
+ }
+
+ private void writeLongBE2(OutputStream output, long val, int wbOffset) {
+ writeBuffer[wbOffset + 0] = (byte) (val >>> 8);
+ writeBuffer[wbOffset + 1] = (byte) (val >>> 0);
+ }
+
+ private void writeLongBE3(OutputStream output, long val, int wbOffset) {
+ writeBuffer[wbOffset + 0] = (byte) (val >>> 16);
+ writeBuffer[wbOffset + 1] = (byte) (val >>> 8);
+ writeBuffer[wbOffset + 2] = (byte) (val >>> 0);
+ }
+
+ private void writeLongBE4(OutputStream output, long val, int wbOffset) {
+ writeBuffer[wbOffset + 0] = (byte) (val >>> 24);
+ writeBuffer[wbOffset + 1] = (byte) (val >>> 16);
+ writeBuffer[wbOffset + 2] = (byte) (val >>> 8);
+ writeBuffer[wbOffset + 3] = (byte) (val >>> 0);
+ }
+
+ private void writeLongBE5(OutputStream output, long val, int wbOffset) {
+ writeBuffer[wbOffset + 0] = (byte) (val >>> 32);
+ writeBuffer[wbOffset + 1] = (byte) (val >>> 24);
+ writeBuffer[wbOffset + 2] = (byte) (val >>> 16);
+ writeBuffer[wbOffset + 3] = (byte) (val >>> 8);
+ writeBuffer[wbOffset + 4] = (byte) (val >>> 0);
+ }
+
+ private void writeLongBE6(OutputStream output, long val, int wbOffset) {
+ writeBuffer[wbOffset + 0] = (byte) (val >>> 40);
+ writeBuffer[wbOffset + 1] = (byte) (val >>> 32);
+ writeBuffer[wbOffset + 2] = (byte) (val >>> 24);
+ writeBuffer[wbOffset + 3] = (byte) (val >>> 16);
+ writeBuffer[wbOffset + 4] = (byte) (val >>> 8);
+ writeBuffer[wbOffset + 5] = (byte) (val >>> 0);
+ }
+
+ private void writeLongBE7(OutputStream output, long val, int wbOffset) {
+ writeBuffer[wbOffset + 0] = (byte) (val >>> 48);
+ writeBuffer[wbOffset + 1] = (byte) (val >>> 40);
+ writeBuffer[wbOffset + 2] = (byte) (val >>> 32);
+ writeBuffer[wbOffset + 3] = (byte) (val >>> 24);
+ writeBuffer[wbOffset + 4] = (byte) (val >>> 16);
+ writeBuffer[wbOffset + 5] = (byte) (val >>> 8);
+ writeBuffer[wbOffset + 6] = (byte) (val >>> 0);
+ }
+
+ private void writeLongBE8(OutputStream output, long val, int wbOffset) {
+ writeBuffer[wbOffset + 0] = (byte) (val >>> 56);
+ writeBuffer[wbOffset + 1] = (byte) (val >>> 48);
+ writeBuffer[wbOffset + 2] = (byte) (val >>> 40);
+ writeBuffer[wbOffset + 3] = (byte) (val >>> 32);
+ writeBuffer[wbOffset + 4] = (byte) (val >>> 24);
+ writeBuffer[wbOffset + 5] = (byte) (val >>> 16);
+ writeBuffer[wbOffset + 6] = (byte) (val >>> 8);
+ writeBuffer[wbOffset + 7] = (byte) (val >>> 0);
+ }
+
+ /**
+ * Read bitpacked integers from input stream
+ * @param buffer - input buffer
+ * @param offset - offset
+ * @param len - length
+ * @param bitSize - bit width
+ * @param input - input stream
+ * @throws IOException
+ */
+ public void readInts(long[] buffer, int offset, int len, int bitSize,
+ InStream input) throws IOException {
+ int bitsLeft = 0;
+ int current = 0;
+
+ switch (bitSize) {
+ case 1:
+ unrolledUnPack1(buffer, offset, len, input);
+ return;
+ case 2:
+ unrolledUnPack2(buffer, offset, len, input);
+ return;
+ case 4:
+ unrolledUnPack4(buffer, offset, len, input);
+ return;
+ case 8:
+ unrolledUnPack8(buffer, offset, len, input);
+ return;
+ case 16:
+ unrolledUnPack16(buffer, offset, len, input);
+ return;
+ case 24:
+ unrolledUnPack24(buffer, offset, len, input);
+ return;
+ case 32:
+ unrolledUnPack32(buffer, offset, len, input);
+ return;
+ case 40:
+ unrolledUnPack40(buffer, offset, len, input);
+ return;
+ case 48:
+ unrolledUnPack48(buffer, offset, len, input);
+ return;
+ case 56:
+ unrolledUnPack56(buffer, offset, len, input);
+ return;
+ case 64:
+ unrolledUnPack64(buffer, offset, len, input);
+ return;
+ default:
+ break;
+ }
+
+ for(int i = offset; i < (offset + len); i++) {
+ long result = 0;
+ int bitsLeftToRead = bitSize;
+ while (bitsLeftToRead > bitsLeft) {
+ result <<= bitsLeft;
+ result |= current & ((1 << bitsLeft) - 1);
+ bitsLeftToRead -= bitsLeft;
+ current = input.read();
+ bitsLeft = 8;
+ }
+
+ // handle the left over bits
+ if (bitsLeftToRead > 0) {
+ result <<= bitsLeftToRead;
+ bitsLeft -= bitsLeftToRead;
+ result |= (current >> bitsLeft) & ((1 << bitsLeftToRead) - 1);
+ }
+ buffer[i] = result;
+ }
+ }
+
+
+ private void unrolledUnPack1(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ final int numHops = 8;
+ final int remainder = len % numHops;
+ final int endOffset = offset + len;
+ final int endUnroll = endOffset - remainder;
+ int val = 0;
+ for (int i = offset; i < endUnroll; i = i + numHops) {
+ val = input.read();
+ buffer[i] = (val >>> 7) & 1;
+ buffer[i + 1] = (val >>> 6) & 1;
+ buffer[i + 2] = (val >>> 5) & 1;
+ buffer[i + 3] = (val >>> 4) & 1;
+ buffer[i + 4] = (val >>> 3) & 1;
+ buffer[i + 5] = (val >>> 2) & 1;
+ buffer[i + 6] = (val >>> 1) & 1;
+ buffer[i + 7] = val & 1;
+ }
+
+ if (remainder > 0) {
+ int startShift = 7;
+ val = input.read();
+ for (int i = endUnroll; i < endOffset; i++) {
+ buffer[i] = (val >>> startShift) & 1;
+ startShift -= 1;
+ }
+ }
+ }
+
+ private void unrolledUnPack2(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ final int numHops = 4;
+ final int remainder = len % numHops;
+ final int endOffset = offset + len;
+ final int endUnroll = endOffset - remainder;
+ int val = 0;
+ for (int i = offset; i < endUnroll; i = i + numHops) {
+ val = input.read();
+ buffer[i] = (val >>> 6) & 3;
+ buffer[i + 1] = (val >>> 4) & 3;
+ buffer[i + 2] = (val >>> 2) & 3;
+ buffer[i + 3] = val & 3;
+ }
+
+ if (remainder > 0) {
+ int startShift = 6;
+ val = input.read();
+ for (int i = endUnroll; i < endOffset; i++) {
+ buffer[i] = (val >>> startShift) & 3;
+ startShift -= 2;
+ }
+ }
+ }
+
+ private void unrolledUnPack4(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ final int numHops = 2;
+ final int remainder = len % numHops;
+ final int endOffset = offset + len;
+ final int endUnroll = endOffset - remainder;
+ int val = 0;
+ for (int i = offset; i < endUnroll; i = i + numHops) {
+ val = input.read();
+ buffer[i] = (val >>> 4) & 15;
+ buffer[i + 1] = val & 15;
+ }
+
+ if (remainder > 0) {
+ int startShift = 4;
+ val = input.read();
+ for (int i = endUnroll; i < endOffset; i++) {
+ buffer[i] = (val >>> startShift) & 15;
+ startShift -= 4;
+ }
+ }
+ }
+
+ private void unrolledUnPack8(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ unrolledUnPackBytes(buffer, offset, len, input, 1);
+ }
+
+ private void unrolledUnPack16(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ unrolledUnPackBytes(buffer, offset, len, input, 2);
+ }
+
+ private void unrolledUnPack24(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ unrolledUnPackBytes(buffer, offset, len, input, 3);
+ }
+
+ private void unrolledUnPack32(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ unrolledUnPackBytes(buffer, offset, len, input, 4);
+ }
+
+ private void unrolledUnPack40(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ unrolledUnPackBytes(buffer, offset, len, input, 5);
+ }
+
+ private void unrolledUnPack48(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ unrolledUnPackBytes(buffer, offset, len, input, 6);
+ }
+
+ private void unrolledUnPack56(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ unrolledUnPackBytes(buffer, offset, len, input, 7);
+ }
+
+ private void unrolledUnPack64(long[] buffer, int offset, int len,
+ InStream input) throws IOException {
+ unrolledUnPackBytes(buffer, offset, len, input, 8);
+ }
+
+ private void unrolledUnPackBytes(long[] buffer, int offset, int len, InStream input, int numBytes)
+ throws IOException {
+ final int numHops = 8;
+ final int remainder = len % numHops;
+ final int endOffset = offset + len;
+ final int endUnroll = endOffset - remainder;
+ int i = offset;
+ for (; i < endUnroll; i = i + numHops) {
+ readLongBE(input, buffer, i, numHops, numBytes);
+ }
+
+ if (remainder > 0) {
+ readRemainingLongs(buffer, i, input, remainder, numBytes);
+ }
+ }
+
+ private void readRemainingLongs(long[] buffer, int offset, InStream input, int remainder,
+ int numBytes) throws IOException {
+ final int toRead = remainder * numBytes;
+ // bulk read to buffer
+ int bytesRead = input.read(readBuffer, 0, toRead);
+ while (bytesRead != toRead) {
+ bytesRead += input.read(readBuffer, bytesRead, toRead - bytesRead);
+ }
+
+ int idx = 0;
+ switch (numBytes) {
+ case 1:
+ while (remainder > 0) {
+ buffer[offset++] = readBuffer[idx] & 255;
+ remainder--;
+ idx++;
+ }
+ break;
+ case 2:
+ while (remainder > 0) {
+ buffer[offset++] = readLongBE2(input, idx * 2);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 3:
+ while (remainder > 0) {
+ buffer[offset++] = readLongBE3(input, idx * 3);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 4:
+ while (remainder > 0) {
+ buffer[offset++] = readLongBE4(input, idx * 4);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 5:
+ while (remainder > 0) {
+ buffer[offset++] = readLongBE5(input, idx * 5);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 6:
+ while (remainder > 0) {
+ buffer[offset++] = readLongBE6(input, idx * 6);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 7:
+ while (remainder > 0) {
+ buffer[offset++] = readLongBE7(input, idx * 7);
+ remainder--;
+ idx++;
+ }
+ break;
+ case 8:
+ while (remainder > 0) {
+ buffer[offset++] = readLongBE8(input, idx * 8);
+ remainder--;
+ idx++;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void readLongBE(InStream in, long[] buffer, int start, int numHops, int numBytes)
+ throws IOException {
+ final int toRead = numHops * numBytes;
+ // bulk read to buffer
+ int bytesRead = in.read(readBuffer, 0, toRead);
+ while (bytesRead != toRead) {
+ bytesRead += in.read(readBuffer, bytesRead, toRead - bytesRead);
+ }
+
+ switch (numBytes) {
+ case 1:
+ buffer[start + 0] = readBuffer[0] & 255;
+ buffer[start + 1] = readBuffer[1] & 255;
+ buffer[start + 2] = readBuffer[2] & 255;
+ buffer[start + 3] = readBuffer[3] & 255;
+ buffer[start + 4] = readBuffer[4] & 255;
+ buffer[start + 5] = readBuffer[5] & 255;
+ buffer[start + 6] = readBuffer[6] & 255;
+ buffer[start + 7] = readBuffer[7] & 255;
+ break;
+ case 2:
+ buffer[start + 0] = readLongBE2(in, 0);
+ buffer[start + 1] = readLongBE2(in, 2);
+ buffer[start + 2] = readLongBE2(in, 4);
+ buffer[start + 3] = readLongBE2(in, 6);
+ buffer[start + 4] = readLongBE2(in, 8);
+ buffer[start + 5] = readLongBE2(in, 10);
+ buffer[start + 6] = readLongBE2(in, 12);
+ buffer[start + 7] = readLongBE2(in, 14);
+ break;
+ case 3:
+ buffer[start + 0] = readLongBE3(in, 0);
+ buffer[start + 1] = readLongBE3(in, 3);
+ buffer[start + 2] = readLongBE3(in, 6);
+ buffer[start + 3] = readLongBE3(in, 9);
+ buffer[start + 4] = readLongBE3(in, 12);
+ buffer[start + 5] = readLongBE3(in, 15);
+ buffer[start + 6] = readLongBE3(in, 18);
+ buffer[start + 7] = readLongBE3(in, 21);
+ break;
+ case 4:
+ buffer[start + 0] = readLongBE4(in, 0);
+ buffer[start + 1] = readLongBE4(in, 4);
+ buffer[start + 2] = readLongBE4(in, 8);
+ buffer[start + 3] = readLongBE4(in, 12);
+ buffer[start + 4] = readLongBE4(in, 16);
+ buffer[start + 5] = readLongBE4(in, 20);
+ buffer[start + 6] = readLongBE4(in, 24);
+ buffer[start + 7] = readLongBE4(in, 28);
+ break;
+ case 5:
+ buffer[start + 0] = readLongBE5(in, 0);
+ buffer[start + 1] = readLongBE5(in, 5);
+ buffer[start + 2] = readLongBE5(in, 10);
+ buffer[start + 3] = readLongBE5(in, 15);
+ buffer[start + 4] = readLongBE5(in, 20);
+ buffer[start + 5] = readLongBE5(in, 25);
+ buffer[start + 6] = readLongBE5(in, 30);
+ buffer[start + 7] = readLongBE5(in, 35);
+ break;
+ case 6:
+ buffer[start + 0] = readLongBE6(in, 0);
+ buffer[start + 1] = readLongBE6(in, 6);
+ buffer[start + 2] = readLongBE6(in, 12);
+ buffer[start + 3] = readLongBE6(in, 18);
+ buffer[start + 4] = readLongBE6(in, 24);
+ buffer[start + 5] = readLongBE6(in, 30);
+ buffer[start + 6] = readLongBE6(in, 36);
+ buffer[start + 7] = readLongBE6(in, 42);
+ break;
+ case 7:
+ buffer[start + 0] = readLongBE7(in, 0);
+ buffer[start + 1] = readLongBE7(in, 7);
+ buffer[start + 2] = readLongBE7(in, 14);
+ buffer[start + 3] = readLongBE7(in, 21);
+ buffer[start + 4] = readLongBE7(in, 28);
+ buffer[start + 5] = readLongBE7(in, 35);
+ buffer[start + 6] = readLongBE7(in, 42);
+ buffer[start + 7] = readLongBE7(in, 49);
+ break;
+ case 8:
+ buffer[start + 0] = readLongBE8(in, 0);
+ buffer[start + 1] = readLongBE8(in, 8);
+ buffer[start + 2] = readLongBE8(in, 16);
+ buffer[start + 3] = readLongBE8(in, 24);
+ buffer[start + 4] = readLongBE8(in, 32);
+ buffer[start + 5] = readLongBE8(in, 40);
+ buffer[start + 6] = readLongBE8(in, 48);
+ buffer[start + 7] = readLongBE8(in, 56);
+ break;
+ default:
+ break;
+ }
+ }
+
+ private long readLongBE2(InStream in, int rbOffset) {
+ return (((readBuffer[rbOffset] & 255) << 8)
+ + ((readBuffer[rbOffset + 1] & 255) << 0));
+ }
+
+ private long readLongBE3(InStream in, int rbOffset) {
+ return (((readBuffer[rbOffset] & 255) << 16)
+ + ((readBuffer[rbOffset + 1] & 255) << 8)
+ + ((readBuffer[rbOffset + 2] & 255) << 0));
+ }
+
+ private long readLongBE4(InStream in, int rbOffset) {
+ return (((long) (readBuffer[rbOffset] & 255) << 24)
+ + ((readBuffer[rbOffset + 1] & 255) << 16)
+ + ((readBuffer[rbOffset + 2] & 255) << 8)
+ + ((readBuffer[rbOffset + 3] & 255) << 0));
+ }
+
+ private long readLongBE5(InStream in, int rbOffset) {
+ return (((long) (readBuffer[rbOffset] & 255) << 32)
+ + ((long) (readBuffer[rbOffset + 1] & 255) << 24)
+ + ((readBuffer[rbOffset + 2] & 255) << 16)
+ + ((readBuffer[rbOffset + 3] & 255) << 8)
+ + ((readBuffer[rbOffset + 4] & 255) << 0));
+ }
+
+ private long readLongBE6(InStream in, int rbOffset) {
+ return (((long) (readBuffer[rbOffset] & 255) << 40)
+ + ((long) (readBuffer[rbOffset + 1] & 255) << 32)
+ + ((long) (readBuffer[rbOffset + 2] & 255) << 24)
+ + ((readBuffer[rbOffset + 3] & 255) << 16)
+ + ((readBuffer[rbOffset + 4] & 255) << 8)
+ + ((readBuffer[rbOffset + 5] & 255) << 0));
+ }
+
+ private long readLongBE7(InStream in, int rbOffset) {
+ return (((long) (readBuffer[rbOffset] & 255) << 48)
+ + ((long) (readBuffer[rbOffset + 1] & 255) << 40)
+ + ((long) (readBuffer[rbOffset + 2] & 255) << 32)
+ + ((long) (readBuffer[rbOffset + 3] & 255) << 24)
+ + ((readBuffer[rbOffset + 4] & 255) << 16)
+ + ((readBuffer[rbOffset + 5] & 255) << 8)
+ + ((readBuffer[rbOffset + 6] & 255) << 0));
+ }
+
+ private long readLongBE8(InStream in, int rbOffset) {
+ return (((long) (readBuffer[rbOffset] & 255) << 56)
+ + ((long) (readBuffer[rbOffset + 1] & 255) << 48)
+ + ((long) (readBuffer[rbOffset + 2] & 255) << 40)
+ + ((long) (readBuffer[rbOffset + 3] & 255) << 32)
+ + ((long) (readBuffer[rbOffset + 4] & 255) << 24)
+ + ((readBuffer[rbOffset + 5] & 255) << 16)
+ + ((readBuffer[rbOffset + 6] & 255) << 8)
+ + ((readBuffer[rbOffset + 7] & 255) << 0));
+ }
+
+ // Do not want to use Guava LongMath.checkedSubtract() here as it will throw
+ // ArithmeticException in case of overflow
+ public boolean isSafeSubtract(long left, long right) {
+ return (left ^ right) >= 0 | (left ^ (left - right)) >= 0;
+ }
+}
[10/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/SettableUncompressedStream.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/SettableUncompressedStream.java b/orc/src/java/org/apache/orc/impl/SettableUncompressedStream.java
new file mode 100644
index 0000000..f9e29eb
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/SettableUncompressedStream.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.common.DiskRangeInfo;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.orc.impl.InStream;
+
+/**
+ * An uncompressed stream whose underlying byte buffer can be set.
+ */
+public class SettableUncompressedStream extends InStream.UncompressedStream {
+
+ public SettableUncompressedStream(String name, List<DiskRange> input, long length) {
+ super(name, input, length);
+ setOffset(input);
+ }
+
+ public void setBuffers(DiskRangeInfo diskRangeInfo) {
+ reset(diskRangeInfo.getDiskRanges(), diskRangeInfo.getTotalLength());
+ setOffset(diskRangeInfo.getDiskRanges());
+ }
+
+ private void setOffset(List<DiskRange> list) {
+ currentOffset = list.isEmpty() ? 0 : list.get(0).getOffset();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/SnappyCodec.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/SnappyCodec.java b/orc/src/java/org/apache/orc/impl/SnappyCodec.java
new file mode 100644
index 0000000..dd4f30c
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/SnappyCodec.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.orc.CompressionCodec;
+import org.iq80.snappy.Snappy;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.EnumSet;
+
+public class SnappyCodec implements CompressionCodec, DirectDecompressionCodec {
+ private static final HadoopShims SHIMS = HadoopShims.Factory.get();
+
+ Boolean direct = null;
+
+ @Override
+ public boolean compress(ByteBuffer in, ByteBuffer out,
+ ByteBuffer overflow) throws IOException {
+ int inBytes = in.remaining();
+ // I should work on a patch for Snappy to support an overflow buffer
+ // to prevent the extra buffer copy.
+ byte[] compressed = new byte[Snappy.maxCompressedLength(inBytes)];
+ int outBytes =
+ Snappy.compress(in.array(), in.arrayOffset() + in.position(), inBytes,
+ compressed, 0);
+ if (outBytes < inBytes) {
+ int remaining = out.remaining();
+ if (remaining >= outBytes) {
+ System.arraycopy(compressed, 0, out.array(), out.arrayOffset() +
+ out.position(), outBytes);
+ out.position(out.position() + outBytes);
+ } else {
+ System.arraycopy(compressed, 0, out.array(), out.arrayOffset() +
+ out.position(), remaining);
+ out.position(out.limit());
+ System.arraycopy(compressed, remaining, overflow.array(),
+ overflow.arrayOffset(), outBytes - remaining);
+ overflow.position(outBytes - remaining);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void decompress(ByteBuffer in, ByteBuffer out) throws IOException {
+ if(in.isDirect() && out.isDirect()) {
+ directDecompress(in, out);
+ return;
+ }
+ int inOffset = in.position();
+ int uncompressLen =
+ Snappy.uncompress(in.array(), in.arrayOffset() + inOffset,
+ in.limit() - inOffset, out.array(), out.arrayOffset() + out.position());
+ out.position(uncompressLen + out.position());
+ out.flip();
+ }
+
+ @Override
+ public boolean isAvailable() {
+ if (direct == null) {
+ try {
+ if (SHIMS.getDirectDecompressor(
+ HadoopShims.DirectCompressionType.SNAPPY) != null) {
+ direct = Boolean.valueOf(true);
+ } else {
+ direct = Boolean.valueOf(false);
+ }
+ } catch (UnsatisfiedLinkError ule) {
+ direct = Boolean.valueOf(false);
+ }
+ }
+ return direct.booleanValue();
+ }
+
+ @Override
+ public void directDecompress(ByteBuffer in, ByteBuffer out)
+ throws IOException {
+ HadoopShims.DirectDecompressor decompressShim =
+ SHIMS.getDirectDecompressor(HadoopShims.DirectCompressionType.SNAPPY);
+ decompressShim.decompress(in, out);
+ out.flip(); // flip for read
+ }
+
+ @Override
+ public CompressionCodec modify(EnumSet<Modifier> modifiers) {
+ // snappy allows no modifications
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/StreamName.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/StreamName.java b/orc/src/java/org/apache/orc/impl/StreamName.java
new file mode 100644
index 0000000..b3fd145
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/StreamName.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.orc.OrcProto;
+
+/**
+ * The name of a stream within a stripe.
+ */
+public class StreamName implements Comparable<StreamName> {
+ private final int column;
+ private final OrcProto.Stream.Kind kind;
+
+ public static enum Area {
+ DATA, INDEX
+ }
+
+ public StreamName(int column, OrcProto.Stream.Kind kind) {
+ this.column = column;
+ this.kind = kind;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj != null && obj instanceof StreamName) {
+ StreamName other = (StreamName) obj;
+ return other.column == column && other.kind == kind;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int compareTo(StreamName streamName) {
+ if (streamName == null) {
+ return -1;
+ }
+ Area area = getArea(kind);
+ Area otherArea = streamName.getArea(streamName.kind);
+ if (area != otherArea) {
+ return -area.compareTo(otherArea);
+ }
+ if (column != streamName.column) {
+ return column < streamName.column ? -1 : 1;
+ }
+ return kind.compareTo(streamName.kind);
+ }
+
+ public int getColumn() {
+ return column;
+ }
+
+ public OrcProto.Stream.Kind getKind() {
+ return kind;
+ }
+
+ public Area getArea() {
+ return getArea(kind);
+ }
+
+ public static Area getArea(OrcProto.Stream.Kind kind) {
+ switch (kind) {
+ case ROW_INDEX:
+ case DICTIONARY_COUNT:
+ case BLOOM_FILTER:
+ return Area.INDEX;
+ default:
+ return Area.DATA;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Stream for column " + column + " kind " + kind;
+ }
+
+ @Override
+ public int hashCode() {
+ return column * 101 + kind.getNumber();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/StringRedBlackTree.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/StringRedBlackTree.java b/orc/src/java/org/apache/orc/impl/StringRedBlackTree.java
new file mode 100644
index 0000000..c353ab0
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/StringRedBlackTree.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.hadoop.io.Text;
+import org.apache.orc.impl.DynamicByteArray;
+import org.apache.orc.impl.DynamicIntArray;
+import org.apache.orc.impl.RedBlackTree;
+
+/**
+ * A red-black tree that stores strings. The strings are stored as UTF-8 bytes
+ * and an offset for each entry.
+ */
+public class StringRedBlackTree extends RedBlackTree {
+ private final DynamicByteArray byteArray = new DynamicByteArray();
+ private final DynamicIntArray keyOffsets;
+ private final Text newKey = new Text();
+
+ public StringRedBlackTree(int initialCapacity) {
+ super(initialCapacity);
+ keyOffsets = new DynamicIntArray(initialCapacity);
+ }
+
+ public int add(String value) {
+ newKey.set(value);
+ return addNewKey();
+ }
+
+ private int addNewKey() {
+ // if the newKey is actually new, add it to our byteArray and store the offset & length
+ if (add()) {
+ int len = newKey.getLength();
+ keyOffsets.add(byteArray.add(newKey.getBytes(), 0, len));
+ }
+ return lastAdd;
+ }
+
+ public int add(Text value) {
+ newKey.set(value);
+ return addNewKey();
+ }
+
+ public int add(byte[] bytes, int offset, int length) {
+ newKey.set(bytes, offset, length);
+ return addNewKey();
+ }
+
+ @Override
+ protected int compareValue(int position) {
+ int start = keyOffsets.get(position);
+ int end;
+ if (position + 1 == keyOffsets.size()) {
+ end = byteArray.size();
+ } else {
+ end = keyOffsets.get(position+1);
+ }
+ return byteArray.compare(newKey.getBytes(), 0, newKey.getLength(),
+ start, end - start);
+ }
+
+ /**
+ * The information about each node.
+ */
+ public interface VisitorContext {
+ /**
+ * Get the position where the key was originally added.
+ * @return the number returned by add.
+ */
+ int getOriginalPosition();
+
+ /**
+ * Write the bytes for the string to the given output stream.
+ * @param out the stream to write to.
+ * @throws IOException
+ */
+ void writeBytes(OutputStream out) throws IOException;
+
+ /**
+ * Get the original string.
+ * @return the string
+ */
+ Text getText();
+
+ /**
+ * Get the number of bytes.
+ * @return the string's length in bytes
+ */
+ int getLength();
+ }
+
+ /**
+ * The interface for visitors.
+ */
+ public interface Visitor {
+ /**
+ * Called once for each node of the tree in sort order.
+ * @param context the information about each node
+ * @throws IOException
+ */
+ void visit(VisitorContext context) throws IOException;
+ }
+
+ private class VisitorContextImpl implements VisitorContext {
+ private int originalPosition;
+ private int start;
+ private int end;
+ private final Text text = new Text();
+
+ public int getOriginalPosition() {
+ return originalPosition;
+ }
+
+ public Text getText() {
+ byteArray.setText(text, start, end - start);
+ return text;
+ }
+
+ public void writeBytes(OutputStream out) throws IOException {
+ byteArray.write(out, start, end - start);
+ }
+
+ public int getLength() {
+ return end - start;
+ }
+
+ void setPosition(int position) {
+ originalPosition = position;
+ start = keyOffsets.get(originalPosition);
+ if (position + 1 == keyOffsets.size()) {
+ end = byteArray.size();
+ } else {
+ end = keyOffsets.get(originalPosition + 1);
+ }
+ }
+ }
+
+ private void recurse(int node, Visitor visitor, VisitorContextImpl context
+ ) throws IOException {
+ if (node != NULL) {
+ recurse(getLeft(node), visitor, context);
+ context.setPosition(node);
+ visitor.visit(context);
+ recurse(getRight(node), visitor, context);
+ }
+ }
+
+ /**
+ * Visit all of the nodes in the tree in sorted order.
+ * @param visitor the action to be applied to each node
+ * @throws IOException
+ */
+ public void visit(Visitor visitor) throws IOException {
+ recurse(root, visitor, new VisitorContextImpl());
+ }
+
+ /**
+ * Reset the table to empty.
+ */
+ public void clear() {
+ super.clear();
+ byteArray.clear();
+ keyOffsets.clear();
+ }
+
+ public void getText(Text result, int originalPosition) {
+ int offset = keyOffsets.get(originalPosition);
+ int length;
+ if (originalPosition + 1 == keyOffsets.size()) {
+ length = byteArray.size() - offset;
+ } else {
+ length = keyOffsets.get(originalPosition + 1) - offset;
+ }
+ byteArray.setText(result, offset, length);
+ }
+
+ /**
+ * Get the size of the character data in the table.
+ * @return the bytes used by the table
+ */
+ public int getCharacterSize() {
+ return byteArray.size();
+ }
+
+ /**
+ * Calculate the approximate size in memory.
+ * @return the number of bytes used in storing the tree.
+ */
+ public long getSizeInBytes() {
+ return byteArray.getSizeInBytes() + keyOffsets.getSizeInBytes() +
+ super.getSizeInBytes();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/ZlibCodec.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/ZlibCodec.java b/orc/src/java/org/apache/orc/impl/ZlibCodec.java
new file mode 100644
index 0000000..5f648a8
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/ZlibCodec.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.EnumSet;
+import java.util.zip.DataFormatException;
+import java.util.zip.Deflater;
+import java.util.zip.Inflater;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.io.compress.DirectDecompressor;
+import org.apache.orc.CompressionCodec;
+
+public class ZlibCodec implements CompressionCodec, DirectDecompressionCodec {
+ private static final HadoopShims SHIMS = HadoopShims.Factory.get();
+ private Boolean direct = null;
+
+ private final int level;
+ private final int strategy;
+
+ public ZlibCodec() {
+ level = Deflater.DEFAULT_COMPRESSION;
+ strategy = Deflater.DEFAULT_STRATEGY;
+ }
+
+ private ZlibCodec(int level, int strategy) {
+ this.level = level;
+ this.strategy = strategy;
+ }
+
+ @Override
+ public boolean compress(ByteBuffer in, ByteBuffer out,
+ ByteBuffer overflow) throws IOException {
+ Deflater deflater = new Deflater(level, true);
+ deflater.setStrategy(strategy);
+ int length = in.remaining();
+ deflater.setInput(in.array(), in.arrayOffset() + in.position(), length);
+ deflater.finish();
+ int outSize = 0;
+ int offset = out.arrayOffset() + out.position();
+ while (!deflater.finished() && (length > outSize)) {
+ int size = deflater.deflate(out.array(), offset, out.remaining());
+ out.position(size + out.position());
+ outSize += size;
+ offset += size;
+ // if we run out of space in the out buffer, use the overflow
+ if (out.remaining() == 0) {
+ if (overflow == null) {
+ deflater.end();
+ return false;
+ }
+ out = overflow;
+ offset = out.arrayOffset() + out.position();
+ }
+ }
+ deflater.end();
+ return length > outSize;
+ }
+
+ @Override
+ public void decompress(ByteBuffer in, ByteBuffer out) throws IOException {
+
+ if(in.isDirect() && out.isDirect()) {
+ directDecompress(in, out);
+ return;
+ }
+
+ Inflater inflater = new Inflater(true);
+ inflater.setInput(in.array(), in.arrayOffset() + in.position(),
+ in.remaining());
+ while (!(inflater.finished() || inflater.needsDictionary() ||
+ inflater.needsInput())) {
+ try {
+ int count = inflater.inflate(out.array(),
+ out.arrayOffset() + out.position(),
+ out.remaining());
+ out.position(count + out.position());
+ } catch (DataFormatException dfe) {
+ throw new IOException("Bad compression data", dfe);
+ }
+ }
+ out.flip();
+ inflater.end();
+ in.position(in.limit());
+ }
+
+ @Override
+ public boolean isAvailable() {
+ if (direct == null) {
+ // see nowrap option in new Inflater(boolean) which disables zlib headers
+ try {
+ if (SHIMS.getDirectDecompressor(
+ HadoopShims.DirectCompressionType.ZLIB_NOHEADER) != null) {
+ direct = Boolean.valueOf(true);
+ } else {
+ direct = Boolean.valueOf(false);
+ }
+ } catch (UnsatisfiedLinkError ule) {
+ direct = Boolean.valueOf(false);
+ }
+ }
+ return direct.booleanValue();
+ }
+
+ @Override
+ public void directDecompress(ByteBuffer in, ByteBuffer out)
+ throws IOException {
+ HadoopShims.DirectDecompressor decompressShim =
+ SHIMS.getDirectDecompressor(HadoopShims.DirectCompressionType.ZLIB_NOHEADER);
+ decompressShim.decompress(in, out);
+ out.flip(); // flip for read
+ }
+
+ @Override
+ public CompressionCodec modify(@Nullable EnumSet<Modifier> modifiers) {
+
+ if (modifiers == null) {
+ return this;
+ }
+
+ int l = this.level;
+ int s = this.strategy;
+
+ for (Modifier m : modifiers) {
+ switch (m) {
+ case BINARY:
+ /* filtered == less LZ77, more huffman */
+ s = Deflater.FILTERED;
+ break;
+ case TEXT:
+ s = Deflater.DEFAULT_STRATEGY;
+ break;
+ case FASTEST:
+ // deflate_fast looking for 8 byte patterns
+ l = Deflater.BEST_SPEED;
+ break;
+ case FAST:
+ // deflate_fast looking for 16 byte patterns
+ l = Deflater.BEST_SPEED + 1;
+ break;
+ case DEFAULT:
+ // deflate_slow looking for 128 byte patterns
+ l = Deflater.DEFAULT_COMPRESSION;
+ break;
+ default:
+ break;
+ }
+ }
+ return new ZlibCodec(l, s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/protobuf/orc_proto.proto
----------------------------------------------------------------------
diff --git a/orc/src/protobuf/orc_proto.proto b/orc/src/protobuf/orc_proto.proto
new file mode 100644
index 0000000..0b36794
--- /dev/null
+++ b/orc/src/protobuf/orc_proto.proto
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package orc.proto;
+
+option java_package = "org.apache.orc";
+
+message IntegerStatistics {
+ optional sint64 minimum = 1;
+ optional sint64 maximum = 2;
+ optional sint64 sum = 3;
+}
+
+message DoubleStatistics {
+ optional double minimum = 1;
+ optional double maximum = 2;
+ optional double sum = 3;
+}
+
+message StringStatistics {
+ optional string minimum = 1;
+ optional string maximum = 2;
+ // sum will store the total length of all strings in a stripe
+ optional sint64 sum = 3;
+}
+
+message BucketStatistics {
+ repeated uint64 count = 1 [packed=true];
+}
+
+message DecimalStatistics {
+ optional string minimum = 1;
+ optional string maximum = 2;
+ optional string sum = 3;
+}
+
+message DateStatistics {
+ // min,max values saved as days since epoch
+ optional sint32 minimum = 1;
+ optional sint32 maximum = 2;
+}
+
+message TimestampStatistics {
+ // min,max values saved as milliseconds since epoch
+ optional sint64 minimum = 1;
+ optional sint64 maximum = 2;
+}
+
+message BinaryStatistics {
+ // sum will store the total binary blob length in a stripe
+ optional sint64 sum = 1;
+}
+
+message ColumnStatistics {
+ optional uint64 numberOfValues = 1;
+ optional IntegerStatistics intStatistics = 2;
+ optional DoubleStatistics doubleStatistics = 3;
+ optional StringStatistics stringStatistics = 4;
+ optional BucketStatistics bucketStatistics = 5;
+ optional DecimalStatistics decimalStatistics = 6;
+ optional DateStatistics dateStatistics = 7;
+ optional BinaryStatistics binaryStatistics = 8;
+ optional TimestampStatistics timestampStatistics = 9;
+ optional bool hasNull = 10;
+}
+
+message RowIndexEntry {
+ repeated uint64 positions = 1 [packed=true];
+ optional ColumnStatistics statistics = 2;
+}
+
+message RowIndex {
+ repeated RowIndexEntry entry = 1;
+}
+
+message BloomFilter {
+ optional uint32 numHashFunctions = 1;
+ repeated fixed64 bitset = 2;
+}
+
+message BloomFilterIndex {
+ repeated BloomFilter bloomFilter = 1;
+}
+
+message Stream {
+ // if you add new index stream kinds, you need to make sure to update
+ // StreamName to ensure it is added to the stripe in the right area
+ enum Kind {
+ PRESENT = 0;
+ DATA = 1;
+ LENGTH = 2;
+ DICTIONARY_DATA = 3;
+ DICTIONARY_COUNT = 4;
+ SECONDARY = 5;
+ ROW_INDEX = 6;
+ BLOOM_FILTER = 7;
+ }
+ optional Kind kind = 1;
+ optional uint32 column = 2;
+ optional uint64 length = 3;
+}
+
+message ColumnEncoding {
+ enum Kind {
+ DIRECT = 0;
+ DICTIONARY = 1;
+ DIRECT_V2 = 2;
+ DICTIONARY_V2 = 3;
+ }
+ optional Kind kind = 1;
+ optional uint32 dictionarySize = 2;
+}
+
+message StripeFooter {
+ repeated Stream streams = 1;
+ repeated ColumnEncoding columns = 2;
+ optional string writerTimezone = 3;
+}
+
+message Type {
+ enum Kind {
+ BOOLEAN = 0;
+ BYTE = 1;
+ SHORT = 2;
+ INT = 3;
+ LONG = 4;
+ FLOAT = 5;
+ DOUBLE = 6;
+ STRING = 7;
+ BINARY = 8;
+ TIMESTAMP = 9;
+ LIST = 10;
+ MAP = 11;
+ STRUCT = 12;
+ UNION = 13;
+ DECIMAL = 14;
+ DATE = 15;
+ VARCHAR = 16;
+ CHAR = 17;
+ }
+ optional Kind kind = 1;
+ repeated uint32 subtypes = 2 [packed=true];
+ repeated string fieldNames = 3;
+ optional uint32 maximumLength = 4;
+ optional uint32 precision = 5;
+ optional uint32 scale = 6;
+}
+
+message StripeInformation {
+ optional uint64 offset = 1;
+ optional uint64 indexLength = 2;
+ optional uint64 dataLength = 3;
+ optional uint64 footerLength = 4;
+ optional uint64 numberOfRows = 5;
+}
+
+message UserMetadataItem {
+ optional string name = 1;
+ optional bytes value = 2;
+}
+
+message StripeStatistics {
+ repeated ColumnStatistics colStats = 1;
+}
+
+message Metadata {
+ repeated StripeStatistics stripeStats = 1;
+}
+
+message Footer {
+ optional uint64 headerLength = 1;
+ optional uint64 contentLength = 2;
+ repeated StripeInformation stripes = 3;
+ repeated Type types = 4;
+ repeated UserMetadataItem metadata = 5;
+ optional uint64 numberOfRows = 6;
+ repeated ColumnStatistics statistics = 7;
+ optional uint32 rowIndexStride = 8;
+}
+
+enum CompressionKind {
+ NONE = 0;
+ ZLIB = 1;
+ SNAPPY = 2;
+ LZO = 3;
+}
+
+// Serialized length must be less that 255 bytes
+message PostScript {
+ optional uint64 footerLength = 1;
+ optional CompressionKind compression = 2;
+ optional uint64 compressionBlockSize = 3;
+ // the version of the file format
+ // [0, 11] = Hive 0.11
+ // [0, 12] = Hive 0.12
+ repeated uint32 version = 4 [packed = true];
+ optional uint64 metadataLength = 5;
+ // Version of the writer:
+ // 0 (or missing) = original
+ // 1 = HIVE-8732 fixed
+ // 2 = HIVE-4243 fixed
+ optional uint32 writerVersion = 6;
+ // Leave this last in the record
+ optional string magic = 8000;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestBitFieldReader.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestBitFieldReader.java b/orc/src/test/org/apache/orc/impl/TestBitFieldReader.java
new file mode 100644
index 0000000..e4c6f6b
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestBitFieldReader.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import static junit.framework.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+
+import org.apache.orc.CompressionCodec;
+import org.junit.Test;
+
+public class TestBitFieldReader {
+
+ public void runSeekTest(CompressionCodec codec) throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ final int COUNT = 16384;
+ BitFieldWriter out = new BitFieldWriter(
+ new OutStream("test", 500, codec, collect), 1);
+ TestInStream.PositionCollector[] positions =
+ new TestInStream.PositionCollector[COUNT];
+ for(int i=0; i < COUNT; ++i) {
+ positions[i] = new TestInStream.PositionCollector();
+ out.getPosition(positions[i]);
+ // test runs, non-runs
+ if (i < COUNT / 2) {
+ out.write(i & 1);
+ } else {
+ out.write((i/3) & 1);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ BitFieldReader in = new BitFieldReader(InStream.create("test",
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
+ codec, 500), 1);
+ for(int i=0; i < COUNT; ++i) {
+ int x = in.next();
+ if (i < COUNT / 2) {
+ assertEquals(i & 1, x);
+ } else {
+ assertEquals((i/3) & 1, x);
+ }
+ }
+ for(int i=COUNT-1; i >= 0; --i) {
+ in.seek(positions[i]);
+ int x = in.next();
+ if (i < COUNT / 2) {
+ assertEquals(i & 1, x);
+ } else {
+ assertEquals((i/3) & 1, x);
+ }
+ }
+ }
+
+ @Test
+ public void testUncompressedSeek() throws Exception {
+ runSeekTest(null);
+ }
+
+ @Test
+ public void testCompressedSeek() throws Exception {
+ runSeekTest(new ZlibCodec());
+ }
+
+ @Test
+ public void testBiggerItems() throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ final int COUNT = 16384;
+ BitFieldWriter out = new BitFieldWriter(
+ new OutStream("test", 500, null, collect), 3);
+ for(int i=0; i < COUNT; ++i) {
+ // test runs, non-runs
+ if (i < COUNT / 2) {
+ out.write(i & 7);
+ } else {
+ out.write((i/3) & 7);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ BitFieldReader in = new BitFieldReader(InStream.create("test",
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
+ null, 500), 3);
+ for(int i=0; i < COUNT; ++i) {
+ int x = in.next();
+ if (i < COUNT / 2) {
+ assertEquals(i & 7, x);
+ } else {
+ assertEquals((i/3) & 7, x);
+ }
+ }
+ }
+
+ @Test
+ public void testSkips() throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ BitFieldWriter out = new BitFieldWriter(
+ new OutStream("test", 100, null, collect), 1);
+ final int COUNT = 16384;
+ for(int i=0; i < COUNT; ++i) {
+ if (i < COUNT/2) {
+ out.write(i & 1);
+ } else {
+ out.write((i/3) & 1);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ BitFieldReader in = new BitFieldReader(InStream.create("test", new ByteBuffer[]{inBuf},
+ new long[]{0}, inBuf.remaining(), null, 100), 1);
+ for(int i=0; i < COUNT; i += 5) {
+ int x = (int) in.next();
+ if (i < COUNT/2) {
+ assertEquals(i & 1, x);
+ } else {
+ assertEquals((i/3) & 1, x);
+ }
+ if (i < COUNT - 5) {
+ in.skip(4);
+ }
+ in.skip(0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestBitPack.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestBitPack.java b/orc/src/test/org/apache/orc/impl/TestBitPack.java
new file mode 100644
index 0000000..f2d3d64
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestBitPack.java
@@ -0,0 +1,279 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import com.google.common.primitives.Longs;
+
+public class TestBitPack {
+
+ private static final int SIZE = 100;
+ private static Random rand = new Random(100);
+ Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test"
+ + File.separator + "tmp"));
+
+ Configuration conf;
+ FileSystem fs;
+ Path testFilePath;
+
+ @Rule
+ public TestName testCaseName = new TestName();
+
+ @Before
+ public void openFileSystem() throws Exception {
+ conf = new Configuration();
+ fs = FileSystem.getLocal(conf);
+ testFilePath = new Path(workDir, "TestOrcFile." + testCaseName.getMethodName() + ".orc");
+ fs.delete(testFilePath, false);
+ }
+
+ private long[] deltaEncode(long[] inp) {
+ long[] output = new long[inp.length];
+ SerializationUtils utils = new SerializationUtils();
+ for (int i = 0; i < inp.length; i++) {
+ output[i] = utils.zigzagEncode(inp[i]);
+ }
+ return output;
+ }
+
+ private long nextLong(Random rng, long n) {
+ long bits, val;
+ do {
+ bits = (rng.nextLong() << 1) >>> 1;
+ val = bits % n;
+ } while (bits - val + (n - 1) < 0L);
+ return val;
+ }
+
+ private void runTest(int numBits) throws IOException {
+ long[] inp = new long[SIZE];
+ for (int i = 0; i < SIZE; i++) {
+ long val = 0;
+ if (numBits <= 32) {
+ if (numBits == 1) {
+ val = -1 * rand.nextInt(2);
+ } else {
+ val = rand.nextInt((int) Math.pow(2, numBits - 1));
+ }
+ } else {
+ val = nextLong(rand, (long) Math.pow(2, numBits - 2));
+ }
+ if (val % 2 == 0) {
+ val = -val;
+ }
+ inp[i] = val;
+ }
+ long[] deltaEncoded = deltaEncode(inp);
+ long minInput = Collections.min(Longs.asList(deltaEncoded));
+ long maxInput = Collections.max(Longs.asList(deltaEncoded));
+ long rangeInput = maxInput - minInput;
+ SerializationUtils utils = new SerializationUtils();
+ int fixedWidth = utils.findClosestNumBits(rangeInput);
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ OutStream output = new OutStream("test", SIZE, null, collect);
+ utils.writeInts(deltaEncoded, 0, deltaEncoded.length, fixedWidth, output);
+ output.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ long[] buff = new long[SIZE];
+ utils.readInts(buff, 0, SIZE, fixedWidth, InStream.create("test", new ByteBuffer[] { inBuf },
+ new long[] { 0 }, inBuf.remaining(), null, SIZE));
+ for (int i = 0; i < SIZE; i++) {
+ buff[i] = utils.zigzagDecode(buff[i]);
+ }
+ assertEquals(numBits, fixedWidth);
+ assertArrayEquals(inp, buff);
+ }
+
+ @Test
+ public void test01BitPacking1Bit() throws IOException {
+ runTest(1);
+ }
+
+ @Test
+ public void test02BitPacking2Bit() throws IOException {
+ runTest(2);
+ }
+
+ @Test
+ public void test03BitPacking3Bit() throws IOException {
+ runTest(3);
+ }
+
+ @Test
+ public void test04BitPacking4Bit() throws IOException {
+ runTest(4);
+ }
+
+ @Test
+ public void test05BitPacking5Bit() throws IOException {
+ runTest(5);
+ }
+
+ @Test
+ public void test06BitPacking6Bit() throws IOException {
+ runTest(6);
+ }
+
+ @Test
+ public void test07BitPacking7Bit() throws IOException {
+ runTest(7);
+ }
+
+ @Test
+ public void test08BitPacking8Bit() throws IOException {
+ runTest(8);
+ }
+
+ @Test
+ public void test09BitPacking9Bit() throws IOException {
+ runTest(9);
+ }
+
+ @Test
+ public void test10BitPacking10Bit() throws IOException {
+ runTest(10);
+ }
+
+ @Test
+ public void test11BitPacking11Bit() throws IOException {
+ runTest(11);
+ }
+
+ @Test
+ public void test12BitPacking12Bit() throws IOException {
+ runTest(12);
+ }
+
+ @Test
+ public void test13BitPacking13Bit() throws IOException {
+ runTest(13);
+ }
+
+ @Test
+ public void test14BitPacking14Bit() throws IOException {
+ runTest(14);
+ }
+
+ @Test
+ public void test15BitPacking15Bit() throws IOException {
+ runTest(15);
+ }
+
+ @Test
+ public void test16BitPacking16Bit() throws IOException {
+ runTest(16);
+ }
+
+ @Test
+ public void test17BitPacking17Bit() throws IOException {
+ runTest(17);
+ }
+
+ @Test
+ public void test18BitPacking18Bit() throws IOException {
+ runTest(18);
+ }
+
+ @Test
+ public void test19BitPacking19Bit() throws IOException {
+ runTest(19);
+ }
+
+ @Test
+ public void test20BitPacking20Bit() throws IOException {
+ runTest(20);
+ }
+
+ @Test
+ public void test21BitPacking21Bit() throws IOException {
+ runTest(21);
+ }
+
+ @Test
+ public void test22BitPacking22Bit() throws IOException {
+ runTest(22);
+ }
+
+ @Test
+ public void test23BitPacking23Bit() throws IOException {
+ runTest(23);
+ }
+
+ @Test
+ public void test24BitPacking24Bit() throws IOException {
+ runTest(24);
+ }
+
+ @Test
+ public void test26BitPacking26Bit() throws IOException {
+ runTest(26);
+ }
+
+ @Test
+ public void test28BitPacking28Bit() throws IOException {
+ runTest(28);
+ }
+
+ @Test
+ public void test30BitPacking30Bit() throws IOException {
+ runTest(30);
+ }
+
+ @Test
+ public void test32BitPacking32Bit() throws IOException {
+ runTest(32);
+ }
+
+ @Test
+ public void test40BitPacking40Bit() throws IOException {
+ runTest(40);
+ }
+
+ @Test
+ public void test48BitPacking48Bit() throws IOException {
+ runTest(48);
+ }
+
+ @Test
+ public void test56BitPacking56Bit() throws IOException {
+ runTest(56);
+ }
+
+ @Test
+ public void test64BitPacking64Bit() throws IOException {
+ runTest(64);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestDynamicArray.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestDynamicArray.java b/orc/src/test/org/apache/orc/impl/TestDynamicArray.java
new file mode 100644
index 0000000..af583f7
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestDynamicArray.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.util.Random;
+
+import org.apache.orc.impl.DynamicByteArray;
+import org.apache.orc.impl.DynamicIntArray;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestDynamicArray {
+
+ @Test
+ public void testByteArray() throws Exception {
+ DynamicByteArray dba = new DynamicByteArray(3, 10);
+ dba.add((byte) 0);
+ dba.add((byte) 1);
+ dba.set(3, (byte) 3);
+ dba.set(2, (byte) 2);
+ dba.add((byte) 4);
+ assertEquals("{0,1,2,3,4}", dba.toString());
+ assertEquals(5, dba.size());
+ byte[] val;
+ val = new byte[0];
+ assertEquals(0, dba.compare(val, 0, 0, 2, 0));
+ assertEquals(-1, dba.compare(val, 0, 0, 2, 1));
+ val = new byte[]{3,42};
+ assertEquals(1, dba.compare(val, 0, 1, 2, 0));
+ assertEquals(1, dba.compare(val, 0, 1, 2, 1));
+ assertEquals(0, dba.compare(val, 0, 1, 3, 1));
+ assertEquals(-1, dba.compare(val, 0, 1, 3, 2));
+ assertEquals(1, dba.compare(val, 0, 2, 3, 1));
+ val = new byte[256];
+ for(int b=-128; b < 128; ++b) {
+ dba.add((byte) b);
+ val[b+128] = (byte) b;
+ }
+ assertEquals(0, dba.compare(val, 0, 256, 5, 256));
+ assertEquals(1, dba.compare(val, 0, 1, 0, 1));
+ assertEquals(1, dba.compare(val, 254, 1, 0, 1));
+ assertEquals(1, dba.compare(val, 120, 1, 64, 1));
+ val = new byte[1024];
+ Random rand = new Random(1701);
+ for(int i = 0; i < val.length; ++i) {
+ rand.nextBytes(val);
+ }
+ dba.add(val, 0, 1024);
+ assertEquals(1285, dba.size());
+ assertEquals(0, dba.compare(val, 0, 1024, 261, 1024));
+ }
+
+ @Test
+ public void testIntArray() throws Exception {
+ DynamicIntArray dia = new DynamicIntArray(10);
+ for(int i=0; i < 10000; ++i) {
+ dia.add(2*i);
+ }
+ assertEquals(10000, dia.size());
+ for(int i=0; i < 10000; ++i) {
+ assertEquals(2*i, dia.get(i));
+ }
+ dia.clear();
+ assertEquals(0, dia.size());
+ dia.add(3);
+ dia.add(12);
+ dia.add(65);
+ assertEquals("{3,12,65}", dia.toString());
+ for(int i=0; i < 5; ++i) {
+ dia.increment(i, 3);
+ }
+ assertEquals("{6,15,68,3,3}", dia.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestInStream.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestInStream.java b/orc/src/test/org/apache/orc/impl/TestInStream.java
new file mode 100644
index 0000000..9e65345
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestInStream.java
@@ -0,0 +1,314 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.fail;
+
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.orc.CompressionCodec;
+import org.junit.Test;
+
+public class TestInStream {
+
+ static class OutputCollector implements OutStream.OutputReceiver {
+ DynamicByteArray buffer = new DynamicByteArray();
+
+ @Override
+ public void output(ByteBuffer buffer) throws IOException {
+ this.buffer.add(buffer.array(), buffer.arrayOffset() + buffer.position(),
+ buffer.remaining());
+ }
+ }
+
+ static class PositionCollector
+ implements PositionProvider, PositionRecorder {
+ private List<Long> positions = new ArrayList<Long>();
+ private int index = 0;
+
+ @Override
+ public long getNext() {
+ return positions.get(index++);
+ }
+
+ @Override
+ public void addPosition(long offset) {
+ positions.add(offset);
+ }
+
+ public void reset() {
+ index = 0;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder("position: ");
+ for(int i=0; i < positions.size(); ++i) {
+ if (i != 0) {
+ builder.append(", ");
+ }
+ builder.append(positions.get(i));
+ }
+ return builder.toString();
+ }
+ }
+
+ @Test
+ public void testUncompressed() throws Exception {
+ OutputCollector collect = new OutputCollector();
+ OutStream out = new OutStream("test", 100, null, collect);
+ PositionCollector[] positions = new PositionCollector[1024];
+ for(int i=0; i < 1024; ++i) {
+ positions[i] = new PositionCollector();
+ out.getPosition(positions[i]);
+ out.write(i);
+ }
+ out.flush();
+ assertEquals(1024, collect.buffer.size());
+ for(int i=0; i < 1024; ++i) {
+ assertEquals((byte) i, collect.buffer.get(i));
+ }
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ InStream in = InStream.create("test", new ByteBuffer[]{inBuf},
+ new long[]{0}, inBuf.remaining(), null, 100);
+ assertEquals("uncompressed stream test position: 0 length: 1024" +
+ " range: 0 offset: 0 limit: 0",
+ in.toString());
+ for(int i=0; i < 1024; ++i) {
+ int x = in.read();
+ assertEquals(i & 0xff, x);
+ }
+ for(int i=1023; i >= 0; --i) {
+ in.seek(positions[i]);
+ assertEquals(i & 0xff, in.read());
+ }
+ }
+
+ @Test
+ public void testCompressed() throws Exception {
+ OutputCollector collect = new OutputCollector();
+ CompressionCodec codec = new ZlibCodec();
+ OutStream out = new OutStream("test", 300, codec, collect);
+ PositionCollector[] positions = new PositionCollector[1024];
+ for(int i=0; i < 1024; ++i) {
+ positions[i] = new PositionCollector();
+ out.getPosition(positions[i]);
+ out.write(i);
+ }
+ out.flush();
+ assertEquals("test", out.toString());
+ assertEquals(961, collect.buffer.size());
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ InStream in = InStream.create("test", new ByteBuffer[]{inBuf},
+ new long[]{0}, inBuf.remaining(), codec, 300);
+ assertEquals("compressed stream test position: 0 length: 961 range: 0" +
+ " offset: 0 limit: 0 range 0 = 0 to 961",
+ in.toString());
+ for(int i=0; i < 1024; ++i) {
+ int x = in.read();
+ assertEquals(i & 0xff, x);
+ }
+ assertEquals(0, in.available());
+ for(int i=1023; i >= 0; --i) {
+ in.seek(positions[i]);
+ assertEquals(i & 0xff, in.read());
+ }
+ }
+
+ @Test
+ public void testCorruptStream() throws Exception {
+ OutputCollector collect = new OutputCollector();
+ CompressionCodec codec = new ZlibCodec();
+ OutStream out = new OutStream("test", 500, codec, collect);
+ PositionCollector[] positions = new PositionCollector[1024];
+ for(int i=0; i < 1024; ++i) {
+ positions[i] = new PositionCollector();
+ out.getPosition(positions[i]);
+ out.write(i);
+ }
+ out.flush();
+
+ // now try to read the stream with a buffer that is too small
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ InStream in = InStream.create("test", new ByteBuffer[]{inBuf},
+ new long[]{0}, inBuf.remaining(), codec, 100);
+ byte[] contents = new byte[1024];
+ try {
+ in.read(contents);
+ fail();
+ } catch(IllegalArgumentException iae) {
+ // EXPECTED
+ }
+
+ // make a corrupted header
+ inBuf.clear();
+ inBuf.put((byte) 32);
+ inBuf.put((byte) 0);
+ inBuf.flip();
+ in = InStream.create("test2", new ByteBuffer[]{inBuf}, new long[]{0},
+ inBuf.remaining(), codec, 300);
+ try {
+ in.read();
+ fail();
+ } catch (IllegalStateException ise) {
+ // EXPECTED
+ }
+ }
+
+ @Test
+ public void testDisjointBuffers() throws Exception {
+ OutputCollector collect = new OutputCollector();
+ CompressionCodec codec = new ZlibCodec();
+ OutStream out = new OutStream("test", 400, codec, collect);
+ PositionCollector[] positions = new PositionCollector[1024];
+ DataOutput stream = new DataOutputStream(out);
+ for(int i=0; i < 1024; ++i) {
+ positions[i] = new PositionCollector();
+ out.getPosition(positions[i]);
+ stream.writeInt(i);
+ }
+ out.flush();
+ assertEquals("test", out.toString());
+ assertEquals(1674, collect.buffer.size());
+ ByteBuffer[] inBuf = new ByteBuffer[3];
+ inBuf[0] = ByteBuffer.allocate(500);
+ inBuf[1] = ByteBuffer.allocate(1200);
+ inBuf[2] = ByteBuffer.allocate(500);
+ collect.buffer.setByteBuffer(inBuf[0], 0, 483);
+ collect.buffer.setByteBuffer(inBuf[1], 483, 1625 - 483);
+ collect.buffer.setByteBuffer(inBuf[2], 1625, 1674 - 1625);
+
+ for(int i=0; i < inBuf.length; ++i) {
+ inBuf[i].flip();
+ }
+ InStream in = InStream.create("test", inBuf,
+ new long[]{0,483, 1625}, 1674, codec, 400);
+ assertEquals("compressed stream test position: 0 length: 1674 range: 0" +
+ " offset: 0 limit: 0 range 0 = 0 to 483;" +
+ " range 1 = 483 to 1142; range 2 = 1625 to 49",
+ in.toString());
+ DataInputStream inStream = new DataInputStream(in);
+ for(int i=0; i < 1024; ++i) {
+ int x = inStream.readInt();
+ assertEquals(i, x);
+ }
+ assertEquals(0, in.available());
+ for(int i=1023; i >= 0; --i) {
+ in.seek(positions[i]);
+ assertEquals(i, inStream.readInt());
+ }
+
+ in = InStream.create("test", new ByteBuffer[]{inBuf[1], inBuf[2]},
+ new long[]{483, 1625}, 1674, codec, 400);
+ inStream = new DataInputStream(in);
+ positions[303].reset();
+ in.seek(positions[303]);
+ for(int i=303; i < 1024; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+
+ in = InStream.create("test", new ByteBuffer[]{inBuf[0], inBuf[2]},
+ new long[]{0, 1625}, 1674, codec, 400);
+ inStream = new DataInputStream(in);
+ positions[1001].reset();
+ for(int i=0; i < 300; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+ in.seek(positions[1001]);
+ for(int i=1001; i < 1024; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+ }
+
+ @Test
+ public void testUncompressedDisjointBuffers() throws Exception {
+ OutputCollector collect = new OutputCollector();
+ OutStream out = new OutStream("test", 400, null, collect);
+ PositionCollector[] positions = new PositionCollector[1024];
+ DataOutput stream = new DataOutputStream(out);
+ for(int i=0; i < 1024; ++i) {
+ positions[i] = new PositionCollector();
+ out.getPosition(positions[i]);
+ stream.writeInt(i);
+ }
+ out.flush();
+ assertEquals("test", out.toString());
+ assertEquals(4096, collect.buffer.size());
+ ByteBuffer[] inBuf = new ByteBuffer[3];
+ inBuf[0] = ByteBuffer.allocate(1100);
+ inBuf[1] = ByteBuffer.allocate(2200);
+ inBuf[2] = ByteBuffer.allocate(1100);
+ collect.buffer.setByteBuffer(inBuf[0], 0, 1024);
+ collect.buffer.setByteBuffer(inBuf[1], 1024, 2048);
+ collect.buffer.setByteBuffer(inBuf[2], 3072, 1024);
+
+ for(int i=0; i < inBuf.length; ++i) {
+ inBuf[i].flip();
+ }
+ InStream in = InStream.create("test", inBuf,
+ new long[]{0, 1024, 3072}, 4096, null, 400);
+ assertEquals("uncompressed stream test position: 0 length: 4096" +
+ " range: 0 offset: 0 limit: 0",
+ in.toString());
+ DataInputStream inStream = new DataInputStream(in);
+ for(int i=0; i < 1024; ++i) {
+ int x = inStream.readInt();
+ assertEquals(i, x);
+ }
+ assertEquals(0, in.available());
+ for(int i=1023; i >= 0; --i) {
+ in.seek(positions[i]);
+ assertEquals(i, inStream.readInt());
+ }
+
+ in = InStream.create("test", new ByteBuffer[]{inBuf[1], inBuf[2]},
+ new long[]{1024, 3072}, 4096, null, 400);
+ inStream = new DataInputStream(in);
+ positions[256].reset();
+ in.seek(positions[256]);
+ for(int i=256; i < 1024; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+
+ in = InStream.create("test", new ByteBuffer[]{inBuf[0], inBuf[2]},
+ new long[]{0, 3072}, 4096, null, 400);
+ inStream = new DataInputStream(in);
+ positions[768].reset();
+ for(int i=0; i < 256; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+ in.seek(positions[768]);
+ for(int i=768; i < 1024; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestIntegerCompressionReader.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestIntegerCompressionReader.java b/orc/src/test/org/apache/orc/impl/TestIntegerCompressionReader.java
new file mode 100644
index 0000000..399f35e
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestIntegerCompressionReader.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import static junit.framework.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import org.apache.orc.CompressionCodec;
+import org.junit.Test;
+
+public class TestIntegerCompressionReader {
+
+ public void runSeekTest(CompressionCodec codec) throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ RunLengthIntegerWriterV2 out = new RunLengthIntegerWriterV2(
+ new OutStream("test", 1000, codec, collect), true);
+ TestInStream.PositionCollector[] positions =
+ new TestInStream.PositionCollector[4096];
+ Random random = new Random(99);
+ int[] junk = new int[2048];
+ for(int i=0; i < junk.length; ++i) {
+ junk[i] = random.nextInt();
+ }
+ for(int i=0; i < 4096; ++i) {
+ positions[i] = new TestInStream.PositionCollector();
+ out.getPosition(positions[i]);
+ // test runs, incrementing runs, non-runs
+ if (i < 1024) {
+ out.write(i/4);
+ } else if (i < 2048) {
+ out.write(2*i);
+ } else {
+ out.write(junk[i-2048]);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ RunLengthIntegerReaderV2 in =
+ new RunLengthIntegerReaderV2(InStream.create
+ ("test", new ByteBuffer[]{inBuf},
+ new long[]{0}, inBuf.remaining(),
+ codec, 1000), true, false);
+ for(int i=0; i < 2048; ++i) {
+ int x = (int) in.next();
+ if (i < 1024) {
+ assertEquals(i/4, x);
+ } else if (i < 2048) {
+ assertEquals(2*i, x);
+ } else {
+ assertEquals(junk[i-2048], x);
+ }
+ }
+ for(int i=2047; i >= 0; --i) {
+ in.seek(positions[i]);
+ int x = (int) in.next();
+ if (i < 1024) {
+ assertEquals(i/4, x);
+ } else if (i < 2048) {
+ assertEquals(2*i, x);
+ } else {
+ assertEquals(junk[i-2048], x);
+ }
+ }
+ }
+
+ @Test
+ public void testUncompressedSeek() throws Exception {
+ runSeekTest(null);
+ }
+
+ @Test
+ public void testCompressedSeek() throws Exception {
+ runSeekTest(new ZlibCodec());
+ }
+
+ @Test
+ public void testSkips() throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ RunLengthIntegerWriterV2 out = new RunLengthIntegerWriterV2(
+ new OutStream("test", 100, null, collect), true);
+ for(int i=0; i < 2048; ++i) {
+ if (i < 1024) {
+ out.write(i);
+ } else {
+ out.write(256 * i);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ RunLengthIntegerReaderV2 in =
+ new RunLengthIntegerReaderV2(InStream.create("test",
+ new ByteBuffer[]{inBuf},
+ new long[]{0},
+ inBuf.remaining(),
+ null, 100), true, false);
+ for(int i=0; i < 2048; i += 10) {
+ int x = (int) in.next();
+ if (i < 1024) {
+ assertEquals(i, x);
+ } else {
+ assertEquals(256 * i, x);
+ }
+ if (i < 2038) {
+ in.skip(9);
+ }
+ in.skip(0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestMemoryManager.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestMemoryManager.java b/orc/src/test/org/apache/orc/impl/TestMemoryManager.java
new file mode 100644
index 0000000..f48c545
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestMemoryManager.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.orc.impl.MemoryManager;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.junit.Test;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+
+import java.lang.management.ManagementFactory;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Test the ORC memory manager.
+ */
+public class TestMemoryManager {
+ private static final double ERROR = 0.000001;
+
+ private static class NullCallback implements MemoryManager.Callback {
+ public boolean checkMemory(double newScale) {
+ return false;
+ }
+ }
+
+ @Test
+ public void testBasics() throws Exception {
+ Configuration conf = new Configuration();
+ MemoryManager mgr = new MemoryManager(conf);
+ NullCallback callback = new NullCallback();
+ long poolSize = mgr.getTotalMemoryPool();
+ assertEquals(Math.round(ManagementFactory.getMemoryMXBean().
+ getHeapMemoryUsage().getMax() * 0.5d), poolSize);
+ assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
+ mgr.addWriter(new Path("p1"), 1000, callback);
+ assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
+ mgr.addWriter(new Path("p1"), poolSize / 2, callback);
+ assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
+ mgr.addWriter(new Path("p2"), poolSize / 2, callback);
+ assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
+ mgr.addWriter(new Path("p3"), poolSize / 2, callback);
+ assertEquals(0.6666667, mgr.getAllocationScale(), 0.00001);
+ mgr.addWriter(new Path("p4"), poolSize / 2, callback);
+ assertEquals(0.5, mgr.getAllocationScale(), 0.000001);
+ mgr.addWriter(new Path("p4"), 3 * poolSize / 2, callback);
+ assertEquals(0.3333333, mgr.getAllocationScale(), 0.000001);
+ mgr.removeWriter(new Path("p1"));
+ mgr.removeWriter(new Path("p2"));
+ assertEquals(0.5, mgr.getAllocationScale(), 0.00001);
+ mgr.removeWriter(new Path("p4"));
+ assertEquals(1.0, mgr.getAllocationScale(), 0.00001);
+ }
+
+ @Test
+ public void testConfig() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("hive.exec.orc.memory.pool", "0.9");
+ MemoryManager mgr = new MemoryManager(conf);
+ long mem =
+ ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+ System.err.print("Memory = " + mem);
+ long pool = mgr.getTotalMemoryPool();
+ assertTrue("Pool too small: " + pool, mem * 0.899 < pool);
+ assertTrue("Pool too big: " + pool, pool < mem * 0.901);
+ }
+
+ private static class DoubleMatcher extends BaseMatcher<Double> {
+ final double expected;
+ final double error;
+ DoubleMatcher(double expected, double error) {
+ this.expected = expected;
+ this.error = error;
+ }
+
+ @Override
+ public boolean matches(Object val) {
+ double dbl = (Double) val;
+ return Math.abs(dbl - expected) <= error;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("not sufficiently close to ");
+ description.appendText(Double.toString(expected));
+ }
+ }
+
+ private static DoubleMatcher closeTo(double value, double error) {
+ return new DoubleMatcher(value, error);
+ }
+
+ @Test
+ public void testCallback() throws Exception {
+ Configuration conf = new Configuration();
+ MemoryManager mgr = new MemoryManager(conf);
+ long pool = mgr.getTotalMemoryPool();
+ MemoryManager.Callback[] calls = new MemoryManager.Callback[20];
+ for(int i=0; i < calls.length; ++i) {
+ calls[i] = Mockito.mock(MemoryManager.Callback.class);
+ mgr.addWriter(new Path(Integer.toString(i)), pool/4, calls[i]);
+ }
+ // add enough rows to get the memory manager to check the limits
+ for(int i=0; i < 10000; ++i) {
+ mgr.addedRow(1);
+ }
+ for(int call=0; call < calls.length; ++call) {
+ Mockito.verify(calls[call], Mockito.times(2))
+ .checkMemory(Matchers.doubleThat(closeTo(0.2, ERROR)));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestRunLengthByteReader.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestRunLengthByteReader.java b/orc/src/test/org/apache/orc/impl/TestRunLengthByteReader.java
new file mode 100644
index 0000000..a14bef1
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestRunLengthByteReader.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import static junit.framework.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+
+import org.apache.orc.CompressionCodec;
+import org.junit.Test;
+
+public class TestRunLengthByteReader {
+
+ @Test
+ public void testUncompressedSeek() throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ RunLengthByteWriter out = new RunLengthByteWriter(new OutStream("test", 100,
+ null, collect));
+ TestInStream.PositionCollector[] positions =
+ new TestInStream.PositionCollector[2048];
+ for(int i=0; i < 2048; ++i) {
+ positions[i] = new TestInStream.PositionCollector();
+ out.getPosition(positions[i]);
+ if (i < 1024) {
+ out.write((byte) (i/4));
+ } else {
+ out.write((byte) i);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), null, 100));
+ for(int i=0; i < 2048; ++i) {
+ int x = in.next() & 0xff;
+ if (i < 1024) {
+ assertEquals((i/4) & 0xff, x);
+ } else {
+ assertEquals(i & 0xff, x);
+ }
+ }
+ for(int i=2047; i >= 0; --i) {
+ in.seek(positions[i]);
+ int x = in.next() & 0xff;
+ if (i < 1024) {
+ assertEquals((i/4) & 0xff, x);
+ } else {
+ assertEquals(i & 0xff, x);
+ }
+ }
+ }
+
+ @Test
+ public void testCompressedSeek() throws Exception {
+ CompressionCodec codec = new SnappyCodec();
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ RunLengthByteWriter out = new RunLengthByteWriter(new OutStream("test", 500,
+ codec, collect));
+ TestInStream.PositionCollector[] positions =
+ new TestInStream.PositionCollector[2048];
+ for(int i=0; i < 2048; ++i) {
+ positions[i] = new TestInStream.PositionCollector();
+ out.getPosition(positions[i]);
+ if (i < 1024) {
+ out.write((byte) (i/4));
+ } else {
+ out.write((byte) i);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), codec, 500));
+ for(int i=0; i < 2048; ++i) {
+ int x = in.next() & 0xff;
+ if (i < 1024) {
+ assertEquals((i/4) & 0xff, x);
+ } else {
+ assertEquals(i & 0xff, x);
+ }
+ }
+ for(int i=2047; i >= 0; --i) {
+ in.seek(positions[i]);
+ int x = in.next() & 0xff;
+ if (i < 1024) {
+ assertEquals((i/4) & 0xff, x);
+ } else {
+ assertEquals(i & 0xff, x);
+ }
+ }
+ }
+
+ @Test
+ public void testSkips() throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ RunLengthByteWriter out = new RunLengthByteWriter(new OutStream("test", 100,
+ null, collect));
+ for(int i=0; i < 2048; ++i) {
+ if (i < 1024) {
+ out.write((byte) (i/16));
+ } else {
+ out.write((byte) i);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), null, 100));
+ for(int i=0; i < 2048; i += 10) {
+ int x = in.next() & 0xff;
+ if (i < 1024) {
+ assertEquals((i/16) & 0xff, x);
+ } else {
+ assertEquals(i & 0xff, x);
+ }
+ if (i < 2038) {
+ in.skip(9);
+ }
+ in.skip(0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestRunLengthIntegerReader.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestRunLengthIntegerReader.java b/orc/src/test/org/apache/orc/impl/TestRunLengthIntegerReader.java
new file mode 100644
index 0000000..28239ba
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestRunLengthIntegerReader.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import static junit.framework.Assert.assertEquals;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import org.apache.orc.CompressionCodec;
+import org.junit.Test;
+
+public class TestRunLengthIntegerReader {
+
+ public void runSeekTest(CompressionCodec codec) throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ RunLengthIntegerWriter out = new RunLengthIntegerWriter(
+ new OutStream("test", 1000, codec, collect), true);
+ TestInStream.PositionCollector[] positions =
+ new TestInStream.PositionCollector[4096];
+ Random random = new Random(99);
+ int[] junk = new int[2048];
+ for(int i=0; i < junk.length; ++i) {
+ junk[i] = random.nextInt();
+ }
+ for(int i=0; i < 4096; ++i) {
+ positions[i] = new TestInStream.PositionCollector();
+ out.getPosition(positions[i]);
+ // test runs, incrementing runs, non-runs
+ if (i < 1024) {
+ out.write(i/4);
+ } else if (i < 2048) {
+ out.write(2*i);
+ } else {
+ out.write(junk[i-2048]);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ RunLengthIntegerReader in = new RunLengthIntegerReader(InStream.create
+ ("test", new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
+ codec, 1000), true);
+ for(int i=0; i < 2048; ++i) {
+ int x = (int) in.next();
+ if (i < 1024) {
+ assertEquals(i/4, x);
+ } else if (i < 2048) {
+ assertEquals(2*i, x);
+ } else {
+ assertEquals(junk[i-2048], x);
+ }
+ }
+ for(int i=2047; i >= 0; --i) {
+ in.seek(positions[i]);
+ int x = (int) in.next();
+ if (i < 1024) {
+ assertEquals(i/4, x);
+ } else if (i < 2048) {
+ assertEquals(2*i, x);
+ } else {
+ assertEquals(junk[i-2048], x);
+ }
+ }
+ }
+
+ @Test
+ public void testUncompressedSeek() throws Exception {
+ runSeekTest(null);
+ }
+
+ @Test
+ public void testCompressedSeek() throws Exception {
+ runSeekTest(new ZlibCodec());
+ }
+
+ @Test
+ public void testSkips() throws Exception {
+ TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+ RunLengthIntegerWriter out = new RunLengthIntegerWriter(
+ new OutStream("test", 100, null, collect), true);
+ for(int i=0; i < 2048; ++i) {
+ if (i < 1024) {
+ out.write(i);
+ } else {
+ out.write(256 * i);
+ }
+ }
+ out.flush();
+ ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+ collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+ inBuf.flip();
+ RunLengthIntegerReader in = new RunLengthIntegerReader(InStream.create
+ ("test", new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
+ null, 100), true);
+ for(int i=0; i < 2048; i += 10) {
+ int x = (int) in.next();
+ if (i < 1024) {
+ assertEquals(i, x);
+ } else {
+ assertEquals(256 * i, x);
+ }
+ if (i < 2038) {
+ in.skip(9);
+ }
+ in.skip(0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/test/org/apache/orc/impl/TestSerializationUtils.java
----------------------------------------------------------------------
diff --git a/orc/src/test/org/apache/orc/impl/TestSerializationUtils.java b/orc/src/test/org/apache/orc/impl/TestSerializationUtils.java
new file mode 100644
index 0000000..0785412
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestSerializationUtils.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.math.BigInteger;
+
+import org.junit.Test;
+
+import com.google.common.math.LongMath;
+
+public class TestSerializationUtils {
+
+ private InputStream fromBuffer(ByteArrayOutputStream buffer) {
+ return new ByteArrayInputStream(buffer.toByteArray());
+ }
+
+ @Test
+ public void testDoubles() throws Exception {
+ double tolerance = 0.0000000000000001;
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ SerializationUtils utils = new SerializationUtils();
+ utils.writeDouble(buffer, 1343822337.759);
+ assertEquals(1343822337.759, utils.readDouble(fromBuffer(buffer)), tolerance);
+ buffer = new ByteArrayOutputStream();
+ utils.writeDouble(buffer, 0.8);
+ double got = utils.readDouble(fromBuffer(buffer));
+ assertEquals(0.8, got, tolerance);
+ }
+
+ @Test
+ public void testBigIntegers() throws Exception {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(0));
+ assertArrayEquals(new byte[]{0}, buffer.toByteArray());
+ assertEquals(0L,
+ SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
+ buffer.reset();
+ SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(1));
+ assertArrayEquals(new byte[]{2}, buffer.toByteArray());
+ assertEquals(1L,
+ SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
+ buffer.reset();
+ SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(-1));
+ assertArrayEquals(new byte[]{1}, buffer.toByteArray());
+ assertEquals(-1L,
+ SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
+ buffer.reset();
+ SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(50));
+ assertArrayEquals(new byte[]{100}, buffer.toByteArray());
+ assertEquals(50L,
+ SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
+ buffer.reset();
+ SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(-50));
+ assertArrayEquals(new byte[]{99}, buffer.toByteArray());
+ assertEquals(-50L,
+ SerializationUtils.readBigInteger(fromBuffer(buffer)).longValue());
+ for(int i=-8192; i < 8192; ++i) {
+ buffer.reset();
+ SerializationUtils.writeBigInteger(buffer, BigInteger.valueOf(i));
+ assertEquals("compare length for " + i,
+ i >= -64 && i < 64 ? 1 : 2, buffer.size());
+ assertEquals("compare result for " + i,
+ i, SerializationUtils.readBigInteger(fromBuffer(buffer)).intValue());
+ }
+ buffer.reset();
+ SerializationUtils.writeBigInteger(buffer,
+ new BigInteger("123456789abcdef0",16));
+ assertEquals(new BigInteger("123456789abcdef0",16),
+ SerializationUtils.readBigInteger(fromBuffer(buffer)));
+ buffer.reset();
+ SerializationUtils.writeBigInteger(buffer,
+ new BigInteger("-123456789abcdef0",16));
+ assertEquals(new BigInteger("-123456789abcdef0",16),
+ SerializationUtils.readBigInteger(fromBuffer(buffer)));
+ StringBuilder buf = new StringBuilder();
+ for(int i=0; i < 256; ++i) {
+ String num = Integer.toHexString(i);
+ if (num.length() == 1) {
+ buf.append('0');
+ }
+ buf.append(num);
+ }
+ buffer.reset();
+ SerializationUtils.writeBigInteger(buffer,
+ new BigInteger(buf.toString(),16));
+ assertEquals(new BigInteger(buf.toString(),16),
+ SerializationUtils.readBigInteger(fromBuffer(buffer)));
+ buffer.reset();
+ SerializationUtils.writeBigInteger(buffer,
+ new BigInteger("ff000000000000000000000000000000000000000000ff",16));
+ assertEquals(
+ new BigInteger("ff000000000000000000000000000000000000000000ff",16),
+ SerializationUtils.readBigInteger(fromBuffer(buffer)));
+ }
+
+ @Test
+ public void testSubtractionOverflow() {
+ // cross check results with Guava results below
+ SerializationUtils utils = new SerializationUtils();
+ assertEquals(false, utils.isSafeSubtract(22222222222L, Long.MIN_VALUE));
+ assertEquals(false, utils.isSafeSubtract(-22222222222L, Long.MAX_VALUE));
+ assertEquals(false, utils.isSafeSubtract(Long.MIN_VALUE, Long.MAX_VALUE));
+ assertEquals(true, utils.isSafeSubtract(-1553103058346370095L, 6553103058346370095L));
+ assertEquals(true, utils.isSafeSubtract(0, Long.MAX_VALUE));
+ assertEquals(true, utils.isSafeSubtract(Long.MIN_VALUE, 0));
+ }
+
+ @Test
+ public void testSubtractionOverflowGuava() {
+ try {
+ LongMath.checkedSubtract(22222222222L, Long.MIN_VALUE);
+ fail("expected ArithmeticException for overflow");
+ } catch (ArithmeticException ex) {
+ assertEquals(ex.getMessage(), "overflow");
+ }
+
+ try {
+ LongMath.checkedSubtract(-22222222222L, Long.MAX_VALUE);
+ fail("expected ArithmeticException for overflow");
+ } catch (ArithmeticException ex) {
+ assertEquals(ex.getMessage(), "overflow");
+ }
+
+ try {
+ LongMath.checkedSubtract(Long.MIN_VALUE, Long.MAX_VALUE);
+ fail("expected ArithmeticException for overflow");
+ } catch (ArithmeticException ex) {
+ assertEquals(ex.getMessage(), "overflow");
+ }
+
+ assertEquals(-8106206116692740190L,
+ LongMath.checkedSubtract(-1553103058346370095L, 6553103058346370095L));
+ assertEquals(-Long.MAX_VALUE, LongMath.checkedSubtract(0, Long.MAX_VALUE));
+ assertEquals(Long.MIN_VALUE, LongMath.checkedSubtract(Long.MIN_VALUE, 0));
+ }
+
+ public static void main(String[] args) throws Exception {
+ TestSerializationUtils test = new TestSerializationUtils();
+ test.testDoubles();
+ test.testBigIntegers();
+ }
+}
[03/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
deleted file mode 100644
index 8cc836e..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
+++ /dev/null
@@ -1,1291 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.math.BigInteger;
-
-public final class SerializationUtils {
-
- private final static int BUFFER_SIZE = 64;
- private final byte[] readBuffer;
- private final byte[] writeBuffer;
-
- public SerializationUtils() {
- this.readBuffer = new byte[BUFFER_SIZE];
- this.writeBuffer = new byte[BUFFER_SIZE];
- }
-
- void writeVulong(OutputStream output, long value) throws IOException {
- while (true) {
- if ((value & ~0x7f) == 0) {
- output.write((byte) value);
- return;
- } else {
- output.write((byte) (0x80 | (value & 0x7f)));
- value >>>= 7;
- }
- }
- }
-
- void writeVslong(OutputStream output, long value) throws IOException {
- writeVulong(output, (value << 1) ^ (value >> 63));
- }
-
-
- long readVulong(InputStream in) throws IOException {
- long result = 0;
- long b;
- int offset = 0;
- do {
- b = in.read();
- if (b == -1) {
- throw new EOFException("Reading Vulong past EOF");
- }
- result |= (0x7f & b) << offset;
- offset += 7;
- } while (b >= 0x80);
- return result;
- }
-
- long readVslong(InputStream in) throws IOException {
- long result = readVulong(in);
- return (result >>> 1) ^ -(result & 1);
- }
-
- public float readFloat(InputStream in) throws IOException {
- int ser = in.read() | (in.read() << 8) | (in.read() << 16) |
- (in.read() << 24);
- return Float.intBitsToFloat(ser);
- }
-
- void writeFloat(OutputStream output, float value) throws IOException {
- int ser = Float.floatToIntBits(value);
- output.write(ser & 0xff);
- output.write((ser >> 8) & 0xff);
- output.write((ser >> 16) & 0xff);
- output.write((ser >> 24) & 0xff);
- }
-
- public double readDouble(InputStream in) throws IOException {
- return Double.longBitsToDouble(readLongLE(in));
- }
-
- long readLongLE(InputStream in) throws IOException {
- in.read(readBuffer, 0, 8);
- return (((readBuffer[0] & 0xff) << 0)
- + ((readBuffer[1] & 0xff) << 8)
- + ((readBuffer[2] & 0xff) << 16)
- + ((long) (readBuffer[3] & 0xff) << 24)
- + ((long) (readBuffer[4] & 0xff) << 32)
- + ((long) (readBuffer[5] & 0xff) << 40)
- + ((long) (readBuffer[6] & 0xff) << 48)
- + ((long) (readBuffer[7] & 0xff) << 56));
- }
-
- void writeDouble(OutputStream output, double value) throws IOException {
- writeLongLE(output, Double.doubleToLongBits(value));
- }
-
- private void writeLongLE(OutputStream output, long value) throws IOException {
- writeBuffer[0] = (byte) ((value >> 0) & 0xff);
- writeBuffer[1] = (byte) ((value >> 8) & 0xff);
- writeBuffer[2] = (byte) ((value >> 16) & 0xff);
- writeBuffer[3] = (byte) ((value >> 24) & 0xff);
- writeBuffer[4] = (byte) ((value >> 32) & 0xff);
- writeBuffer[5] = (byte) ((value >> 40) & 0xff);
- writeBuffer[6] = (byte) ((value >> 48) & 0xff);
- writeBuffer[7] = (byte) ((value >> 56) & 0xff);
- output.write(writeBuffer, 0, 8);
- }
-
- /**
- * Write the arbitrarily sized signed BigInteger in vint format.
- *
- * Signed integers are encoded using the low bit as the sign bit using zigzag
- * encoding.
- *
- * Each byte uses the low 7 bits for data and the high bit for stop/continue.
- *
- * Bytes are stored LSB first.
- * @param output the stream to write to
- * @param value the value to output
- * @throws IOException
- */
- static void writeBigInteger(OutputStream output,
- BigInteger value) throws IOException {
- // encode the signed number as a positive integer
- value = value.shiftLeft(1);
- int sign = value.signum();
- if (sign < 0) {
- value = value.negate();
- value = value.subtract(BigInteger.ONE);
- }
- int length = value.bitLength();
- while (true) {
- long lowBits = value.longValue() & 0x7fffffffffffffffL;
- length -= 63;
- // write out the next 63 bits worth of data
- for(int i=0; i < 9; ++i) {
- // if this is the last byte, leave the high bit off
- if (length <= 0 && (lowBits & ~0x7f) == 0) {
- output.write((byte) lowBits);
- return;
- } else {
- output.write((byte) (0x80 | (lowBits & 0x7f)));
- lowBits >>>= 7;
- }
- }
- value = value.shiftRight(63);
- }
- }
-
- /**
- * Read the signed arbitrary sized BigInteger BigInteger in vint format
- * @param input the stream to read from
- * @return the read BigInteger
- * @throws IOException
- */
- static BigInteger readBigInteger(InputStream input) throws IOException {
- BigInteger result = BigInteger.ZERO;
- long work = 0;
- int offset = 0;
- long b;
- do {
- b = input.read();
- if (b == -1) {
- throw new EOFException("Reading BigInteger past EOF from " + input);
- }
- work |= (0x7f & b) << (offset % 63);
- offset += 7;
- // if we've read 63 bits, roll them into the result
- if (offset == 63) {
- result = BigInteger.valueOf(work);
- work = 0;
- } else if (offset % 63 == 0) {
- result = result.or(BigInteger.valueOf(work).shiftLeft(offset-63));
- work = 0;
- }
- } while (b >= 0x80);
- if (work != 0) {
- result = result.or(BigInteger.valueOf(work).shiftLeft((offset/63)*63));
- }
- // convert back to a signed number
- boolean isNegative = result.testBit(0);
- if (isNegative) {
- result = result.add(BigInteger.ONE);
- result = result.negate();
- }
- result = result.shiftRight(1);
- return result;
- }
-
- enum FixedBitSizes {
- ONE, TWO, THREE, FOUR, FIVE, SIX, SEVEN, EIGHT, NINE, TEN, ELEVEN, TWELVE,
- THIRTEEN, FOURTEEN, FIFTEEN, SIXTEEN, SEVENTEEN, EIGHTEEN, NINETEEN,
- TWENTY, TWENTYONE, TWENTYTWO, TWENTYTHREE, TWENTYFOUR, TWENTYSIX,
- TWENTYEIGHT, THIRTY, THIRTYTWO, FORTY, FORTYEIGHT, FIFTYSIX, SIXTYFOUR;
- }
-
- /**
- * Count the number of bits required to encode the given value
- * @param value
- * @return bits required to store value
- */
- int findClosestNumBits(long value) {
- int count = 0;
- while (value != 0) {
- count++;
- value = value >>> 1;
- }
- return getClosestFixedBits(count);
- }
-
- /**
- * zigzag encode the given value
- * @param val
- * @return zigzag encoded value
- */
- long zigzagEncode(long val) {
- return (val << 1) ^ (val >> 63);
- }
-
- /**
- * zigzag decode the given value
- * @param val
- * @return zizag decoded value
- */
- long zigzagDecode(long val) {
- return (val >>> 1) ^ -(val & 1);
- }
-
- /**
- * Compute the bits required to represent pth percentile value
- * @param data - array
- * @param p - percentile value (>=0.0 to <=1.0)
- * @return pth percentile bits
- */
- int percentileBits(long[] data, int offset, int length, double p) {
- if ((p > 1.0) || (p <= 0.0)) {
- return -1;
- }
-
- // histogram that store the encoded bit requirement for each values.
- // maximum number of bits that can encoded is 32 (refer FixedBitSizes)
- int[] hist = new int[32];
-
- // compute the histogram
- for(int i = offset; i < (offset + length); i++) {
- int idx = encodeBitWidth(findClosestNumBits(data[i]));
- hist[idx] += 1;
- }
-
- int perLen = (int) (length * (1.0 - p));
-
- // return the bits required by pth percentile length
- for(int i = hist.length - 1; i >= 0; i--) {
- perLen -= hist[i];
- if (perLen < 0) {
- return decodeBitWidth(i);
- }
- }
-
- return 0;
- }
-
- /**
- * Read n bytes in big endian order and convert to long
- * @param b - byte array
- * @return long value
- */
- long bytesToLongBE(InStream input, int n) throws IOException {
- long out = 0;
- long val = 0;
- while (n > 0) {
- n--;
- // store it in a long and then shift else integer overflow will occur
- val = input.read();
- out |= (val << (n * 8));
- }
- return out;
- }
-
- /**
- * Calculate the number of bytes required
- * @param n - number of values
- * @param numBits - bit width
- * @return number of bytes required
- */
- int getTotalBytesRequired(int n, int numBits) {
- return (n * numBits + 7) / 8;
- }
-
- /**
- * For a given fixed bit this function will return the closest available fixed
- * bit
- * @param n
- * @return closest valid fixed bit
- */
- int getClosestFixedBits(int n) {
- if (n == 0) {
- return 1;
- }
-
- if (n >= 1 && n <= 24) {
- return n;
- } else if (n > 24 && n <= 26) {
- return 26;
- } else if (n > 26 && n <= 28) {
- return 28;
- } else if (n > 28 && n <= 30) {
- return 30;
- } else if (n > 30 && n <= 32) {
- return 32;
- } else if (n > 32 && n <= 40) {
- return 40;
- } else if (n > 40 && n <= 48) {
- return 48;
- } else if (n > 48 && n <= 56) {
- return 56;
- } else {
- return 64;
- }
- }
-
- public int getClosestAlignedFixedBits(int n) {
- if (n == 0 || n == 1) {
- return 1;
- } else if (n > 1 && n <= 2) {
- return 2;
- } else if (n > 2 && n <= 4) {
- return 4;
- } else if (n > 4 && n <= 8) {
- return 8;
- } else if (n > 8 && n <= 16) {
- return 16;
- } else if (n > 16 && n <= 24) {
- return 24;
- } else if (n > 24 && n <= 32) {
- return 32;
- } else if (n > 32 && n <= 40) {
- return 40;
- } else if (n > 40 && n <= 48) {
- return 48;
- } else if (n > 48 && n <= 56) {
- return 56;
- } else {
- return 64;
- }
- }
-
- /**
- * Finds the closest available fixed bit width match and returns its encoded
- * value (ordinal)
- * @param n - fixed bit width to encode
- * @return encoded fixed bit width
- */
- int encodeBitWidth(int n) {
- n = getClosestFixedBits(n);
-
- if (n >= 1 && n <= 24) {
- return n - 1;
- } else if (n > 24 && n <= 26) {
- return FixedBitSizes.TWENTYSIX.ordinal();
- } else if (n > 26 && n <= 28) {
- return FixedBitSizes.TWENTYEIGHT.ordinal();
- } else if (n > 28 && n <= 30) {
- return FixedBitSizes.THIRTY.ordinal();
- } else if (n > 30 && n <= 32) {
- return FixedBitSizes.THIRTYTWO.ordinal();
- } else if (n > 32 && n <= 40) {
- return FixedBitSizes.FORTY.ordinal();
- } else if (n > 40 && n <= 48) {
- return FixedBitSizes.FORTYEIGHT.ordinal();
- } else if (n > 48 && n <= 56) {
- return FixedBitSizes.FIFTYSIX.ordinal();
- } else {
- return FixedBitSizes.SIXTYFOUR.ordinal();
- }
- }
-
- /**
- * Decodes the ordinal fixed bit value to actual fixed bit width value
- * @param n - encoded fixed bit width
- * @return decoded fixed bit width
- */
- int decodeBitWidth(int n) {
- if (n >= FixedBitSizes.ONE.ordinal()
- && n <= FixedBitSizes.TWENTYFOUR.ordinal()) {
- return n + 1;
- } else if (n == FixedBitSizes.TWENTYSIX.ordinal()) {
- return 26;
- } else if (n == FixedBitSizes.TWENTYEIGHT.ordinal()) {
- return 28;
- } else if (n == FixedBitSizes.THIRTY.ordinal()) {
- return 30;
- } else if (n == FixedBitSizes.THIRTYTWO.ordinal()) {
- return 32;
- } else if (n == FixedBitSizes.FORTY.ordinal()) {
- return 40;
- } else if (n == FixedBitSizes.FORTYEIGHT.ordinal()) {
- return 48;
- } else if (n == FixedBitSizes.FIFTYSIX.ordinal()) {
- return 56;
- } else {
- return 64;
- }
- }
-
- /**
- * Bitpack and write the input values to underlying output stream
- * @param input - values to write
- * @param offset - offset
- * @param len - length
- * @param bitSize - bit width
- * @param output - output stream
- * @throws IOException
- */
- void writeInts(long[] input, int offset, int len, int bitSize,
- OutputStream output) throws IOException {
- if (input == null || input.length < 1 || offset < 0 || len < 1
- || bitSize < 1) {
- return;
- }
-
- switch (bitSize) {
- case 1:
- unrolledBitPack1(input, offset, len, output);
- return;
- case 2:
- unrolledBitPack2(input, offset, len, output);
- return;
- case 4:
- unrolledBitPack4(input, offset, len, output);
- return;
- case 8:
- unrolledBitPack8(input, offset, len, output);
- return;
- case 16:
- unrolledBitPack16(input, offset, len, output);
- return;
- case 24:
- unrolledBitPack24(input, offset, len, output);
- return;
- case 32:
- unrolledBitPack32(input, offset, len, output);
- return;
- case 40:
- unrolledBitPack40(input, offset, len, output);
- return;
- case 48:
- unrolledBitPack48(input, offset, len, output);
- return;
- case 56:
- unrolledBitPack56(input, offset, len, output);
- return;
- case 64:
- unrolledBitPack64(input, offset, len, output);
- return;
- default:
- break;
- }
-
- int bitsLeft = 8;
- byte current = 0;
- for(int i = offset; i < (offset + len); i++) {
- long value = input[i];
- int bitsToWrite = bitSize;
- while (bitsToWrite > bitsLeft) {
- // add the bits to the bottom of the current word
- current |= value >>> (bitsToWrite - bitsLeft);
- // subtract out the bits we just added
- bitsToWrite -= bitsLeft;
- // zero out the bits above bitsToWrite
- value &= (1L << bitsToWrite) - 1;
- output.write(current);
- current = 0;
- bitsLeft = 8;
- }
- bitsLeft -= bitsToWrite;
- current |= value << bitsLeft;
- if (bitsLeft == 0) {
- output.write(current);
- current = 0;
- bitsLeft = 8;
- }
- }
-
- // flush
- if (bitsLeft != 8) {
- output.write(current);
- current = 0;
- bitsLeft = 8;
- }
- }
-
- private void unrolledBitPack1(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- final int numHops = 8;
- final int remainder = len % numHops;
- final int endOffset = offset + len;
- final int endUnroll = endOffset - remainder;
- int val = 0;
- for (int i = offset; i < endUnroll; i = i + numHops) {
- val = (int) (val | ((input[i] & 1) << 7)
- | ((input[i + 1] & 1) << 6)
- | ((input[i + 2] & 1) << 5)
- | ((input[i + 3] & 1) << 4)
- | ((input[i + 4] & 1) << 3)
- | ((input[i + 5] & 1) << 2)
- | ((input[i + 6] & 1) << 1)
- | (input[i + 7]) & 1);
- output.write(val);
- val = 0;
- }
-
- if (remainder > 0) {
- int startShift = 7;
- for (int i = endUnroll; i < endOffset; i++) {
- val = (int) (val | (input[i] & 1) << startShift);
- startShift -= 1;
- }
- output.write(val);
- }
- }
-
- private void unrolledBitPack2(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- final int numHops = 4;
- final int remainder = len % numHops;
- final int endOffset = offset + len;
- final int endUnroll = endOffset - remainder;
- int val = 0;
- for (int i = offset; i < endUnroll; i = i + numHops) {
- val = (int) (val | ((input[i] & 3) << 6)
- | ((input[i + 1] & 3) << 4)
- | ((input[i + 2] & 3) << 2)
- | (input[i + 3]) & 3);
- output.write(val);
- val = 0;
- }
-
- if (remainder > 0) {
- int startShift = 6;
- for (int i = endUnroll; i < endOffset; i++) {
- val = (int) (val | (input[i] & 3) << startShift);
- startShift -= 2;
- }
- output.write(val);
- }
- }
-
- private void unrolledBitPack4(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- final int numHops = 2;
- final int remainder = len % numHops;
- final int endOffset = offset + len;
- final int endUnroll = endOffset - remainder;
- int val = 0;
- for (int i = offset; i < endUnroll; i = i + numHops) {
- val = (int) (val | ((input[i] & 15) << 4) | (input[i + 1]) & 15);
- output.write(val);
- val = 0;
- }
-
- if (remainder > 0) {
- int startShift = 4;
- for (int i = endUnroll; i < endOffset; i++) {
- val = (int) (val | (input[i] & 15) << startShift);
- startShift -= 4;
- }
- output.write(val);
- }
- }
-
- private void unrolledBitPack8(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- unrolledBitPackBytes(input, offset, len, output, 1);
- }
-
- private void unrolledBitPack16(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- unrolledBitPackBytes(input, offset, len, output, 2);
- }
-
- private void unrolledBitPack24(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- unrolledBitPackBytes(input, offset, len, output, 3);
- }
-
- private void unrolledBitPack32(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- unrolledBitPackBytes(input, offset, len, output, 4);
- }
-
- private void unrolledBitPack40(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- unrolledBitPackBytes(input, offset, len, output, 5);
- }
-
- private void unrolledBitPack48(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- unrolledBitPackBytes(input, offset, len, output, 6);
- }
-
- private void unrolledBitPack56(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- unrolledBitPackBytes(input, offset, len, output, 7);
- }
-
- private void unrolledBitPack64(long[] input, int offset, int len,
- OutputStream output) throws IOException {
- unrolledBitPackBytes(input, offset, len, output, 8);
- }
-
- private void unrolledBitPackBytes(long[] input, int offset, int len, OutputStream output, int numBytes) throws IOException {
- final int numHops = 8;
- final int remainder = len % numHops;
- final int endOffset = offset + len;
- final int endUnroll = endOffset - remainder;
- int i = offset;
- for (; i < endUnroll; i = i + numHops) {
- writeLongBE(output, input, i, numHops, numBytes);
- }
-
- if (remainder > 0) {
- writeRemainingLongs(output, i, input, remainder, numBytes);
- }
- }
-
- private void writeRemainingLongs(OutputStream output, int offset, long[] input, int remainder,
- int numBytes) throws IOException {
- final int numHops = remainder;
-
- int idx = 0;
- switch (numBytes) {
- case 1:
- while (remainder > 0) {
- writeBuffer[idx] = (byte) (input[offset + idx] & 255);
- remainder--;
- idx++;
- }
- break;
- case 2:
- while (remainder > 0) {
- writeLongBE2(output, input[offset + idx], idx * 2);
- remainder--;
- idx++;
- }
- break;
- case 3:
- while (remainder > 0) {
- writeLongBE3(output, input[offset + idx], idx * 3);
- remainder--;
- idx++;
- }
- break;
- case 4:
- while (remainder > 0) {
- writeLongBE4(output, input[offset + idx], idx * 4);
- remainder--;
- idx++;
- }
- break;
- case 5:
- while (remainder > 0) {
- writeLongBE5(output, input[offset + idx], idx * 5);
- remainder--;
- idx++;
- }
- break;
- case 6:
- while (remainder > 0) {
- writeLongBE6(output, input[offset + idx], idx * 6);
- remainder--;
- idx++;
- }
- break;
- case 7:
- while (remainder > 0) {
- writeLongBE7(output, input[offset + idx], idx * 7);
- remainder--;
- idx++;
- }
- break;
- case 8:
- while (remainder > 0) {
- writeLongBE8(output, input[offset + idx], idx * 8);
- remainder--;
- idx++;
- }
- break;
- default:
- break;
- }
-
- final int toWrite = numHops * numBytes;
- output.write(writeBuffer, 0, toWrite);
- }
-
- private void writeLongBE(OutputStream output, long[] input, int offset, int numHops, int numBytes) throws IOException {
-
- switch (numBytes) {
- case 1:
- writeBuffer[0] = (byte) (input[offset + 0] & 255);
- writeBuffer[1] = (byte) (input[offset + 1] & 255);
- writeBuffer[2] = (byte) (input[offset + 2] & 255);
- writeBuffer[3] = (byte) (input[offset + 3] & 255);
- writeBuffer[4] = (byte) (input[offset + 4] & 255);
- writeBuffer[5] = (byte) (input[offset + 5] & 255);
- writeBuffer[6] = (byte) (input[offset + 6] & 255);
- writeBuffer[7] = (byte) (input[offset + 7] & 255);
- break;
- case 2:
- writeLongBE2(output, input[offset + 0], 0);
- writeLongBE2(output, input[offset + 1], 2);
- writeLongBE2(output, input[offset + 2], 4);
- writeLongBE2(output, input[offset + 3], 6);
- writeLongBE2(output, input[offset + 4], 8);
- writeLongBE2(output, input[offset + 5], 10);
- writeLongBE2(output, input[offset + 6], 12);
- writeLongBE2(output, input[offset + 7], 14);
- break;
- case 3:
- writeLongBE3(output, input[offset + 0], 0);
- writeLongBE3(output, input[offset + 1], 3);
- writeLongBE3(output, input[offset + 2], 6);
- writeLongBE3(output, input[offset + 3], 9);
- writeLongBE3(output, input[offset + 4], 12);
- writeLongBE3(output, input[offset + 5], 15);
- writeLongBE3(output, input[offset + 6], 18);
- writeLongBE3(output, input[offset + 7], 21);
- break;
- case 4:
- writeLongBE4(output, input[offset + 0], 0);
- writeLongBE4(output, input[offset + 1], 4);
- writeLongBE4(output, input[offset + 2], 8);
- writeLongBE4(output, input[offset + 3], 12);
- writeLongBE4(output, input[offset + 4], 16);
- writeLongBE4(output, input[offset + 5], 20);
- writeLongBE4(output, input[offset + 6], 24);
- writeLongBE4(output, input[offset + 7], 28);
- break;
- case 5:
- writeLongBE5(output, input[offset + 0], 0);
- writeLongBE5(output, input[offset + 1], 5);
- writeLongBE5(output, input[offset + 2], 10);
- writeLongBE5(output, input[offset + 3], 15);
- writeLongBE5(output, input[offset + 4], 20);
- writeLongBE5(output, input[offset + 5], 25);
- writeLongBE5(output, input[offset + 6], 30);
- writeLongBE5(output, input[offset + 7], 35);
- break;
- case 6:
- writeLongBE6(output, input[offset + 0], 0);
- writeLongBE6(output, input[offset + 1], 6);
- writeLongBE6(output, input[offset + 2], 12);
- writeLongBE6(output, input[offset + 3], 18);
- writeLongBE6(output, input[offset + 4], 24);
- writeLongBE6(output, input[offset + 5], 30);
- writeLongBE6(output, input[offset + 6], 36);
- writeLongBE6(output, input[offset + 7], 42);
- break;
- case 7:
- writeLongBE7(output, input[offset + 0], 0);
- writeLongBE7(output, input[offset + 1], 7);
- writeLongBE7(output, input[offset + 2], 14);
- writeLongBE7(output, input[offset + 3], 21);
- writeLongBE7(output, input[offset + 4], 28);
- writeLongBE7(output, input[offset + 5], 35);
- writeLongBE7(output, input[offset + 6], 42);
- writeLongBE7(output, input[offset + 7], 49);
- break;
- case 8:
- writeLongBE8(output, input[offset + 0], 0);
- writeLongBE8(output, input[offset + 1], 8);
- writeLongBE8(output, input[offset + 2], 16);
- writeLongBE8(output, input[offset + 3], 24);
- writeLongBE8(output, input[offset + 4], 32);
- writeLongBE8(output, input[offset + 5], 40);
- writeLongBE8(output, input[offset + 6], 48);
- writeLongBE8(output, input[offset + 7], 56);
- break;
- default:
- break;
- }
-
- final int toWrite = numHops * numBytes;
- output.write(writeBuffer, 0, toWrite);
- }
-
- private void writeLongBE2(OutputStream output, long val, int wbOffset) {
- writeBuffer[wbOffset + 0] = (byte) (val >>> 8);
- writeBuffer[wbOffset + 1] = (byte) (val >>> 0);
- }
-
- private void writeLongBE3(OutputStream output, long val, int wbOffset) {
- writeBuffer[wbOffset + 0] = (byte) (val >>> 16);
- writeBuffer[wbOffset + 1] = (byte) (val >>> 8);
- writeBuffer[wbOffset + 2] = (byte) (val >>> 0);
- }
-
- private void writeLongBE4(OutputStream output, long val, int wbOffset) {
- writeBuffer[wbOffset + 0] = (byte) (val >>> 24);
- writeBuffer[wbOffset + 1] = (byte) (val >>> 16);
- writeBuffer[wbOffset + 2] = (byte) (val >>> 8);
- writeBuffer[wbOffset + 3] = (byte) (val >>> 0);
- }
-
- private void writeLongBE5(OutputStream output, long val, int wbOffset) {
- writeBuffer[wbOffset + 0] = (byte) (val >>> 32);
- writeBuffer[wbOffset + 1] = (byte) (val >>> 24);
- writeBuffer[wbOffset + 2] = (byte) (val >>> 16);
- writeBuffer[wbOffset + 3] = (byte) (val >>> 8);
- writeBuffer[wbOffset + 4] = (byte) (val >>> 0);
- }
-
- private void writeLongBE6(OutputStream output, long val, int wbOffset) {
- writeBuffer[wbOffset + 0] = (byte) (val >>> 40);
- writeBuffer[wbOffset + 1] = (byte) (val >>> 32);
- writeBuffer[wbOffset + 2] = (byte) (val >>> 24);
- writeBuffer[wbOffset + 3] = (byte) (val >>> 16);
- writeBuffer[wbOffset + 4] = (byte) (val >>> 8);
- writeBuffer[wbOffset + 5] = (byte) (val >>> 0);
- }
-
- private void writeLongBE7(OutputStream output, long val, int wbOffset) {
- writeBuffer[wbOffset + 0] = (byte) (val >>> 48);
- writeBuffer[wbOffset + 1] = (byte) (val >>> 40);
- writeBuffer[wbOffset + 2] = (byte) (val >>> 32);
- writeBuffer[wbOffset + 3] = (byte) (val >>> 24);
- writeBuffer[wbOffset + 4] = (byte) (val >>> 16);
- writeBuffer[wbOffset + 5] = (byte) (val >>> 8);
- writeBuffer[wbOffset + 6] = (byte) (val >>> 0);
- }
-
- private void writeLongBE8(OutputStream output, long val, int wbOffset) {
- writeBuffer[wbOffset + 0] = (byte) (val >>> 56);
- writeBuffer[wbOffset + 1] = (byte) (val >>> 48);
- writeBuffer[wbOffset + 2] = (byte) (val >>> 40);
- writeBuffer[wbOffset + 3] = (byte) (val >>> 32);
- writeBuffer[wbOffset + 4] = (byte) (val >>> 24);
- writeBuffer[wbOffset + 5] = (byte) (val >>> 16);
- writeBuffer[wbOffset + 6] = (byte) (val >>> 8);
- writeBuffer[wbOffset + 7] = (byte) (val >>> 0);
- }
-
- /**
- * Read bitpacked integers from input stream
- * @param buffer - input buffer
- * @param offset - offset
- * @param len - length
- * @param bitSize - bit width
- * @param input - input stream
- * @throws IOException
- */
- void readInts(long[] buffer, int offset, int len, int bitSize,
- InStream input) throws IOException {
- int bitsLeft = 0;
- int current = 0;
-
- switch (bitSize) {
- case 1:
- unrolledUnPack1(buffer, offset, len, input);
- return;
- case 2:
- unrolledUnPack2(buffer, offset, len, input);
- return;
- case 4:
- unrolledUnPack4(buffer, offset, len, input);
- return;
- case 8:
- unrolledUnPack8(buffer, offset, len, input);
- return;
- case 16:
- unrolledUnPack16(buffer, offset, len, input);
- return;
- case 24:
- unrolledUnPack24(buffer, offset, len, input);
- return;
- case 32:
- unrolledUnPack32(buffer, offset, len, input);
- return;
- case 40:
- unrolledUnPack40(buffer, offset, len, input);
- return;
- case 48:
- unrolledUnPack48(buffer, offset, len, input);
- return;
- case 56:
- unrolledUnPack56(buffer, offset, len, input);
- return;
- case 64:
- unrolledUnPack64(buffer, offset, len, input);
- return;
- default:
- break;
- }
-
- for(int i = offset; i < (offset + len); i++) {
- long result = 0;
- int bitsLeftToRead = bitSize;
- while (bitsLeftToRead > bitsLeft) {
- result <<= bitsLeft;
- result |= current & ((1 << bitsLeft) - 1);
- bitsLeftToRead -= bitsLeft;
- current = input.read();
- bitsLeft = 8;
- }
-
- // handle the left over bits
- if (bitsLeftToRead > 0) {
- result <<= bitsLeftToRead;
- bitsLeft -= bitsLeftToRead;
- result |= (current >> bitsLeft) & ((1 << bitsLeftToRead) - 1);
- }
- buffer[i] = result;
- }
- }
-
-
- private void unrolledUnPack1(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- final int numHops = 8;
- final int remainder = len % numHops;
- final int endOffset = offset + len;
- final int endUnroll = endOffset - remainder;
- int val = 0;
- for (int i = offset; i < endUnroll; i = i + numHops) {
- val = input.read();
- buffer[i] = (val >>> 7) & 1;
- buffer[i + 1] = (val >>> 6) & 1;
- buffer[i + 2] = (val >>> 5) & 1;
- buffer[i + 3] = (val >>> 4) & 1;
- buffer[i + 4] = (val >>> 3) & 1;
- buffer[i + 5] = (val >>> 2) & 1;
- buffer[i + 6] = (val >>> 1) & 1;
- buffer[i + 7] = val & 1;
- }
-
- if (remainder > 0) {
- int startShift = 7;
- val = input.read();
- for (int i = endUnroll; i < endOffset; i++) {
- buffer[i] = (val >>> startShift) & 1;
- startShift -= 1;
- }
- }
- }
-
- private void unrolledUnPack2(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- final int numHops = 4;
- final int remainder = len % numHops;
- final int endOffset = offset + len;
- final int endUnroll = endOffset - remainder;
- int val = 0;
- for (int i = offset; i < endUnroll; i = i + numHops) {
- val = input.read();
- buffer[i] = (val >>> 6) & 3;
- buffer[i + 1] = (val >>> 4) & 3;
- buffer[i + 2] = (val >>> 2) & 3;
- buffer[i + 3] = val & 3;
- }
-
- if (remainder > 0) {
- int startShift = 6;
- val = input.read();
- for (int i = endUnroll; i < endOffset; i++) {
- buffer[i] = (val >>> startShift) & 3;
- startShift -= 2;
- }
- }
- }
-
- private void unrolledUnPack4(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- final int numHops = 2;
- final int remainder = len % numHops;
- final int endOffset = offset + len;
- final int endUnroll = endOffset - remainder;
- int val = 0;
- for (int i = offset; i < endUnroll; i = i + numHops) {
- val = input.read();
- buffer[i] = (val >>> 4) & 15;
- buffer[i + 1] = val & 15;
- }
-
- if (remainder > 0) {
- int startShift = 4;
- val = input.read();
- for (int i = endUnroll; i < endOffset; i++) {
- buffer[i] = (val >>> startShift) & 15;
- startShift -= 4;
- }
- }
- }
-
- private void unrolledUnPack8(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- unrolledUnPackBytes(buffer, offset, len, input, 1);
- }
-
- private void unrolledUnPack16(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- unrolledUnPackBytes(buffer, offset, len, input, 2);
- }
-
- private void unrolledUnPack24(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- unrolledUnPackBytes(buffer, offset, len, input, 3);
- }
-
- private void unrolledUnPack32(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- unrolledUnPackBytes(buffer, offset, len, input, 4);
- }
-
- private void unrolledUnPack40(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- unrolledUnPackBytes(buffer, offset, len, input, 5);
- }
-
- private void unrolledUnPack48(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- unrolledUnPackBytes(buffer, offset, len, input, 6);
- }
-
- private void unrolledUnPack56(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- unrolledUnPackBytes(buffer, offset, len, input, 7);
- }
-
- private void unrolledUnPack64(long[] buffer, int offset, int len,
- InStream input) throws IOException {
- unrolledUnPackBytes(buffer, offset, len, input, 8);
- }
-
- private void unrolledUnPackBytes(long[] buffer, int offset, int len, InStream input, int numBytes)
- throws IOException {
- final int numHops = 8;
- final int remainder = len % numHops;
- final int endOffset = offset + len;
- final int endUnroll = endOffset - remainder;
- int i = offset;
- for (; i < endUnroll; i = i + numHops) {
- readLongBE(input, buffer, i, numHops, numBytes);
- }
-
- if (remainder > 0) {
- readRemainingLongs(buffer, i, input, remainder, numBytes);
- }
- }
-
- private void readRemainingLongs(long[] buffer, int offset, InStream input, int remainder,
- int numBytes) throws IOException {
- final int toRead = remainder * numBytes;
- // bulk read to buffer
- int bytesRead = input.read(readBuffer, 0, toRead);
- while (bytesRead != toRead) {
- bytesRead += input.read(readBuffer, bytesRead, toRead - bytesRead);
- }
-
- int idx = 0;
- switch (numBytes) {
- case 1:
- while (remainder > 0) {
- buffer[offset++] = readBuffer[idx] & 255;
- remainder--;
- idx++;
- }
- break;
- case 2:
- while (remainder > 0) {
- buffer[offset++] = readLongBE2(input, idx * 2);
- remainder--;
- idx++;
- }
- break;
- case 3:
- while (remainder > 0) {
- buffer[offset++] = readLongBE3(input, idx * 3);
- remainder--;
- idx++;
- }
- break;
- case 4:
- while (remainder > 0) {
- buffer[offset++] = readLongBE4(input, idx * 4);
- remainder--;
- idx++;
- }
- break;
- case 5:
- while (remainder > 0) {
- buffer[offset++] = readLongBE5(input, idx * 5);
- remainder--;
- idx++;
- }
- break;
- case 6:
- while (remainder > 0) {
- buffer[offset++] = readLongBE6(input, idx * 6);
- remainder--;
- idx++;
- }
- break;
- case 7:
- while (remainder > 0) {
- buffer[offset++] = readLongBE7(input, idx * 7);
- remainder--;
- idx++;
- }
- break;
- case 8:
- while (remainder > 0) {
- buffer[offset++] = readLongBE8(input, idx * 8);
- remainder--;
- idx++;
- }
- break;
- default:
- break;
- }
- }
-
- private void readLongBE(InStream in, long[] buffer, int start, int numHops, int numBytes)
- throws IOException {
- final int toRead = numHops * numBytes;
- // bulk read to buffer
- int bytesRead = in.read(readBuffer, 0, toRead);
- while (bytesRead != toRead) {
- bytesRead += in.read(readBuffer, bytesRead, toRead - bytesRead);
- }
-
- switch (numBytes) {
- case 1:
- buffer[start + 0] = readBuffer[0] & 255;
- buffer[start + 1] = readBuffer[1] & 255;
- buffer[start + 2] = readBuffer[2] & 255;
- buffer[start + 3] = readBuffer[3] & 255;
- buffer[start + 4] = readBuffer[4] & 255;
- buffer[start + 5] = readBuffer[5] & 255;
- buffer[start + 6] = readBuffer[6] & 255;
- buffer[start + 7] = readBuffer[7] & 255;
- break;
- case 2:
- buffer[start + 0] = readLongBE2(in, 0);
- buffer[start + 1] = readLongBE2(in, 2);
- buffer[start + 2] = readLongBE2(in, 4);
- buffer[start + 3] = readLongBE2(in, 6);
- buffer[start + 4] = readLongBE2(in, 8);
- buffer[start + 5] = readLongBE2(in, 10);
- buffer[start + 6] = readLongBE2(in, 12);
- buffer[start + 7] = readLongBE2(in, 14);
- break;
- case 3:
- buffer[start + 0] = readLongBE3(in, 0);
- buffer[start + 1] = readLongBE3(in, 3);
- buffer[start + 2] = readLongBE3(in, 6);
- buffer[start + 3] = readLongBE3(in, 9);
- buffer[start + 4] = readLongBE3(in, 12);
- buffer[start + 5] = readLongBE3(in, 15);
- buffer[start + 6] = readLongBE3(in, 18);
- buffer[start + 7] = readLongBE3(in, 21);
- break;
- case 4:
- buffer[start + 0] = readLongBE4(in, 0);
- buffer[start + 1] = readLongBE4(in, 4);
- buffer[start + 2] = readLongBE4(in, 8);
- buffer[start + 3] = readLongBE4(in, 12);
- buffer[start + 4] = readLongBE4(in, 16);
- buffer[start + 5] = readLongBE4(in, 20);
- buffer[start + 6] = readLongBE4(in, 24);
- buffer[start + 7] = readLongBE4(in, 28);
- break;
- case 5:
- buffer[start + 0] = readLongBE5(in, 0);
- buffer[start + 1] = readLongBE5(in, 5);
- buffer[start + 2] = readLongBE5(in, 10);
- buffer[start + 3] = readLongBE5(in, 15);
- buffer[start + 4] = readLongBE5(in, 20);
- buffer[start + 5] = readLongBE5(in, 25);
- buffer[start + 6] = readLongBE5(in, 30);
- buffer[start + 7] = readLongBE5(in, 35);
- break;
- case 6:
- buffer[start + 0] = readLongBE6(in, 0);
- buffer[start + 1] = readLongBE6(in, 6);
- buffer[start + 2] = readLongBE6(in, 12);
- buffer[start + 3] = readLongBE6(in, 18);
- buffer[start + 4] = readLongBE6(in, 24);
- buffer[start + 5] = readLongBE6(in, 30);
- buffer[start + 6] = readLongBE6(in, 36);
- buffer[start + 7] = readLongBE6(in, 42);
- break;
- case 7:
- buffer[start + 0] = readLongBE7(in, 0);
- buffer[start + 1] = readLongBE7(in, 7);
- buffer[start + 2] = readLongBE7(in, 14);
- buffer[start + 3] = readLongBE7(in, 21);
- buffer[start + 4] = readLongBE7(in, 28);
- buffer[start + 5] = readLongBE7(in, 35);
- buffer[start + 6] = readLongBE7(in, 42);
- buffer[start + 7] = readLongBE7(in, 49);
- break;
- case 8:
- buffer[start + 0] = readLongBE8(in, 0);
- buffer[start + 1] = readLongBE8(in, 8);
- buffer[start + 2] = readLongBE8(in, 16);
- buffer[start + 3] = readLongBE8(in, 24);
- buffer[start + 4] = readLongBE8(in, 32);
- buffer[start + 5] = readLongBE8(in, 40);
- buffer[start + 6] = readLongBE8(in, 48);
- buffer[start + 7] = readLongBE8(in, 56);
- break;
- default:
- break;
- }
- }
-
- private long readLongBE2(InStream in, int rbOffset) {
- return (((readBuffer[rbOffset] & 255) << 8)
- + ((readBuffer[rbOffset + 1] & 255) << 0));
- }
-
- private long readLongBE3(InStream in, int rbOffset) {
- return (((readBuffer[rbOffset] & 255) << 16)
- + ((readBuffer[rbOffset + 1] & 255) << 8)
- + ((readBuffer[rbOffset + 2] & 255) << 0));
- }
-
- private long readLongBE4(InStream in, int rbOffset) {
- return (((long) (readBuffer[rbOffset] & 255) << 24)
- + ((readBuffer[rbOffset + 1] & 255) << 16)
- + ((readBuffer[rbOffset + 2] & 255) << 8)
- + ((readBuffer[rbOffset + 3] & 255) << 0));
- }
-
- private long readLongBE5(InStream in, int rbOffset) {
- return (((long) (readBuffer[rbOffset] & 255) << 32)
- + ((long) (readBuffer[rbOffset + 1] & 255) << 24)
- + ((readBuffer[rbOffset + 2] & 255) << 16)
- + ((readBuffer[rbOffset + 3] & 255) << 8)
- + ((readBuffer[rbOffset + 4] & 255) << 0));
- }
-
- private long readLongBE6(InStream in, int rbOffset) {
- return (((long) (readBuffer[rbOffset] & 255) << 40)
- + ((long) (readBuffer[rbOffset + 1] & 255) << 32)
- + ((long) (readBuffer[rbOffset + 2] & 255) << 24)
- + ((readBuffer[rbOffset + 3] & 255) << 16)
- + ((readBuffer[rbOffset + 4] & 255) << 8)
- + ((readBuffer[rbOffset + 5] & 255) << 0));
- }
-
- private long readLongBE7(InStream in, int rbOffset) {
- return (((long) (readBuffer[rbOffset] & 255) << 48)
- + ((long) (readBuffer[rbOffset + 1] & 255) << 40)
- + ((long) (readBuffer[rbOffset + 2] & 255) << 32)
- + ((long) (readBuffer[rbOffset + 3] & 255) << 24)
- + ((readBuffer[rbOffset + 4] & 255) << 16)
- + ((readBuffer[rbOffset + 5] & 255) << 8)
- + ((readBuffer[rbOffset + 6] & 255) << 0));
- }
-
- private long readLongBE8(InStream in, int rbOffset) {
- return (((long) (readBuffer[rbOffset] & 255) << 56)
- + ((long) (readBuffer[rbOffset + 1] & 255) << 48)
- + ((long) (readBuffer[rbOffset + 2] & 255) << 40)
- + ((long) (readBuffer[rbOffset + 3] & 255) << 32)
- + ((long) (readBuffer[rbOffset + 4] & 255) << 24)
- + ((readBuffer[rbOffset + 5] & 255) << 16)
- + ((readBuffer[rbOffset + 6] & 255) << 8)
- + ((readBuffer[rbOffset + 7] & 255) << 0));
- }
-
- // Do not want to use Guava LongMath.checkedSubtract() here as it will throw
- // ArithmeticException in case of overflow
- public boolean isSafeSubtract(long left, long right) {
- return (left ^ right) >= 0 | (left ^ (left - right)) >= 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SettableUncompressedStream.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SettableUncompressedStream.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SettableUncompressedStream.java
deleted file mode 100644
index 3496de9..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SettableUncompressedStream.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.common.DiskRangeInfo;
-import org.apache.hadoop.hive.common.io.DiskRange;
-
-/**
- * An uncompressed stream whose underlying byte buffer can be set.
- */
-public class SettableUncompressedStream extends InStream.UncompressedStream {
-
- public SettableUncompressedStream(String name, List<DiskRange> input, long length) {
- super(name, input, length);
- setOffset(input);
- }
-
- public void setBuffers(DiskRangeInfo diskRangeInfo) {
- reset(diskRangeInfo.getDiskRanges(), diskRangeInfo.getTotalLength());
- setOffset(diskRangeInfo.getDiskRanges());
- }
-
- private void setOffset(List<DiskRange> list) {
- currentOffset = list.isEmpty() ? 0 : list.get(0).getOffset();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
deleted file mode 100644
index 820f215..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.apache.hadoop.hive.shims.HadoopShims.DirectDecompressorShim;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectCompressionType;
-import org.iq80.snappy.Snappy;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.EnumSet;
-
-class SnappyCodec implements CompressionCodec, DirectDecompressionCodec {
-
- Boolean direct = null;
-
- @Override
- public boolean compress(ByteBuffer in, ByteBuffer out,
- ByteBuffer overflow) throws IOException {
- int inBytes = in.remaining();
- // I should work on a patch for Snappy to support an overflow buffer
- // to prevent the extra buffer copy.
- byte[] compressed = new byte[Snappy.maxCompressedLength(inBytes)];
- int outBytes =
- Snappy.compress(in.array(), in.arrayOffset() + in.position(), inBytes,
- compressed, 0);
- if (outBytes < inBytes) {
- int remaining = out.remaining();
- if (remaining >= outBytes) {
- System.arraycopy(compressed, 0, out.array(), out.arrayOffset() +
- out.position(), outBytes);
- out.position(out.position() + outBytes);
- } else {
- System.arraycopy(compressed, 0, out.array(), out.arrayOffset() +
- out.position(), remaining);
- out.position(out.limit());
- System.arraycopy(compressed, remaining, overflow.array(),
- overflow.arrayOffset(), outBytes - remaining);
- overflow.position(outBytes - remaining);
- }
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void decompress(ByteBuffer in, ByteBuffer out) throws IOException {
- if(in.isDirect() && out.isDirect()) {
- directDecompress(in, out);
- return;
- }
- int inOffset = in.position();
- int uncompressLen =
- Snappy.uncompress(in.array(), in.arrayOffset() + inOffset,
- in.limit() - inOffset, out.array(), out.arrayOffset() + out.position());
- out.position(uncompressLen + out.position());
- out.flip();
- }
-
- @Override
- public boolean isAvailable() {
- if (direct == null) {
- try {
- if (ShimLoader.getHadoopShims().getDirectDecompressor(
- DirectCompressionType.SNAPPY) != null) {
- direct = Boolean.valueOf(true);
- } else {
- direct = Boolean.valueOf(false);
- }
- } catch (UnsatisfiedLinkError ule) {
- direct = Boolean.valueOf(false);
- }
- }
- return direct.booleanValue();
- }
-
- @Override
- public void directDecompress(ByteBuffer in, ByteBuffer out)
- throws IOException {
- DirectDecompressorShim decompressShim = ShimLoader.getHadoopShims()
- .getDirectDecompressor(DirectCompressionType.SNAPPY);
- decompressShim.decompress(in, out);
- out.flip(); // flip for read
- }
-
- @Override
- public CompressionCodec modify(EnumSet<Modifier> modifiers) {
- // snappy allows no modifications
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java
deleted file mode 100644
index 9ff0b1f..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * The name of a stream within a stripe.
- */
-public class StreamName implements Comparable<StreamName> {
- private final int column;
- private final OrcProto.Stream.Kind kind;
-
- public static enum Area {
- DATA, INDEX
- }
-
- public StreamName(int column, OrcProto.Stream.Kind kind) {
- this.column = column;
- this.kind = kind;
- }
-
- public boolean equals(Object obj) {
- if (obj != null && obj instanceof StreamName) {
- StreamName other = (StreamName) obj;
- return other.column == column && other.kind == kind;
- } else {
- return false;
- }
- }
-
- @Override
- public int compareTo(StreamName streamName) {
- if (streamName == null) {
- return -1;
- }
- Area area = getArea(kind);
- Area otherArea = streamName.getArea(streamName.kind);
- if (area != otherArea) {
- return -area.compareTo(otherArea);
- }
- if (column != streamName.column) {
- return column < streamName.column ? -1 : 1;
- }
- return kind.compareTo(streamName.kind);
- }
-
- public int getColumn() {
- return column;
- }
-
- public OrcProto.Stream.Kind getKind() {
- return kind;
- }
-
- public Area getArea() {
- return getArea(kind);
- }
-
- public static Area getArea(OrcProto.Stream.Kind kind) {
- switch (kind) {
- case ROW_INDEX:
- case DICTIONARY_COUNT:
- case BLOOM_FILTER:
- return Area.INDEX;
- default:
- return Area.DATA;
- }
- }
-
- @Override
- public String toString() {
- return "Stream for column " + column + " kind " + kind;
- }
-
- @Override
- public int hashCode() {
- return column * 101 + kind.getNumber();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java
deleted file mode 100644
index 3a49269..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * Statistics for string columns.
- */
-public interface StringColumnStatistics extends ColumnStatistics {
- /**
- * Get the minimum string.
- * @return the minimum
- */
- String getMinimum();
-
- /**
- * Get the maximum string.
- * @return the maximum
- */
- String getMaximum();
-
- /**
- * Get the total length of all strings
- * @return the sum (total length)
- */
- long getSum();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java
deleted file mode 100644
index e0c52e7..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.apache.hadoop.io.Text;
-
-/**
- * A red-black tree that stores strings. The strings are stored as UTF-8 bytes
- * and an offset for each entry.
- */
-class StringRedBlackTree extends RedBlackTree {
- private final DynamicByteArray byteArray = new DynamicByteArray();
- private final DynamicIntArray keyOffsets;
- private final Text newKey = new Text();
-
- public StringRedBlackTree(int initialCapacity) {
- super(initialCapacity);
- keyOffsets = new DynamicIntArray(initialCapacity);
- }
-
- public int add(String value) {
- newKey.set(value);
- return addNewKey();
- }
-
- private int addNewKey() {
- // if the newKey is actually new, add it to our byteArray and store the offset & length
- if (add()) {
- int len = newKey.getLength();
- keyOffsets.add(byteArray.add(newKey.getBytes(), 0, len));
- }
- return lastAdd;
- }
-
- public int add(Text value) {
- newKey.set(value);
- return addNewKey();
- }
-
- public int add(byte[] bytes, int offset, int length) {
- newKey.set(bytes, offset, length);
- return addNewKey();
- }
-
- @Override
- protected int compareValue(int position) {
- int start = keyOffsets.get(position);
- int end;
- if (position + 1 == keyOffsets.size()) {
- end = byteArray.size();
- } else {
- end = keyOffsets.get(position+1);
- }
- return byteArray.compare(newKey.getBytes(), 0, newKey.getLength(),
- start, end - start);
- }
-
- /**
- * The information about each node.
- */
- public interface VisitorContext {
- /**
- * Get the position where the key was originally added.
- * @return the number returned by add.
- */
- int getOriginalPosition();
-
- /**
- * Write the bytes for the string to the given output stream.
- * @param out the stream to write to.
- * @throws IOException
- */
- void writeBytes(OutputStream out) throws IOException;
-
- /**
- * Get the original string.
- * @return the string
- */
- Text getText();
-
- /**
- * Get the number of bytes.
- * @return the string's length in bytes
- */
- int getLength();
- }
-
- /**
- * The interface for visitors.
- */
- public interface Visitor {
- /**
- * Called once for each node of the tree in sort order.
- * @param context the information about each node
- * @throws IOException
- */
- void visit(VisitorContext context) throws IOException;
- }
-
- private class VisitorContextImpl implements VisitorContext {
- private int originalPosition;
- private int start;
- private int end;
- private final Text text = new Text();
-
- public int getOriginalPosition() {
- return originalPosition;
- }
-
- public Text getText() {
- byteArray.setText(text, start, end - start);
- return text;
- }
-
- public void writeBytes(OutputStream out) throws IOException {
- byteArray.write(out, start, end - start);
- }
-
- public int getLength() {
- return end - start;
- }
-
- void setPosition(int position) {
- originalPosition = position;
- start = keyOffsets.get(originalPosition);
- if (position + 1 == keyOffsets.size()) {
- end = byteArray.size();
- } else {
- end = keyOffsets.get(originalPosition + 1);
- }
- }
- }
-
- private void recurse(int node, Visitor visitor, VisitorContextImpl context
- ) throws IOException {
- if (node != NULL) {
- recurse(getLeft(node), visitor, context);
- context.setPosition(node);
- visitor.visit(context);
- recurse(getRight(node), visitor, context);
- }
- }
-
- /**
- * Visit all of the nodes in the tree in sorted order.
- * @param visitor the action to be applied to each node
- * @throws IOException
- */
- public void visit(Visitor visitor) throws IOException {
- recurse(root, visitor, new VisitorContextImpl());
- }
-
- /**
- * Reset the table to empty.
- */
- public void clear() {
- super.clear();
- byteArray.clear();
- keyOffsets.clear();
- }
-
- public void getText(Text result, int originalPosition) {
- int offset = keyOffsets.get(originalPosition);
- int length;
- if (originalPosition + 1 == keyOffsets.size()) {
- length = byteArray.size() - offset;
- } else {
- length = keyOffsets.get(originalPosition + 1) - offset;
- }
- byteArray.setText(result, offset, length);
- }
-
- /**
- * Get the size of the character data in the table.
- * @return the bytes used by the table
- */
- public int getCharacterSize() {
- return byteArray.size();
- }
-
- /**
- * Calculate the approximate size in memory.
- * @return the number of bytes used in storing the tree.
- */
- public long getSizeInBytes() {
- return byteArray.getSizeInBytes() + keyOffsets.getSizeInBytes() +
- super.getSizeInBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java
deleted file mode 100644
index 2ac4601..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * Information about the stripes in an ORC file that is provided by the Reader.
- */
-public interface StripeInformation {
- /**
- * Get the byte offset of the start of the stripe.
- * @return the bytes from the start of the file
- */
- long getOffset();
-
- /**
- * Get the total length of the stripe in bytes.
- * @return the number of bytes in the stripe
- */
- long getLength();
-
- /**
- * Get the length of the stripe's indexes.
- * @return the number of bytes in the index
- */
- long getIndexLength();
-
- /**
- * Get the length of the stripe's data.
- * @return the number of bytes in the stripe
- */
- long getDataLength();
-
- /**
- * Get the length of the stripe's tail section, which contains its index.
- * @return the number of bytes in the tail
- */
- long getFooterLength();
-
- /**
- * Get the number of rows in the stripe.
- * @return a count of the number of rows
- */
- long getNumberOfRows();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java
deleted file mode 100644
index 6fc1ab6..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.util.List;
-
-public class StripeStatistics {
- private final List<OrcProto.ColumnStatistics> cs;
-
- StripeStatistics(List<OrcProto.ColumnStatistics> list) {
- this.cs = list;
- }
-
- /**
- * Return list of column statistics
- *
- * @return column stats
- */
- public ColumnStatistics[] getColumnStatistics() {
- ColumnStatistics[] result = new ColumnStatistics[cs.size()];
- for (int i = 0; i < result.length; ++i) {
- result[i] = ColumnStatisticsImpl.deserialize(cs.get(i));
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TimestampColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TimestampColumnStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TimestampColumnStatistics.java
deleted file mode 100644
index ef42f50..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TimestampColumnStatistics.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.sql.Timestamp;
-
-/**
- * Statistics for Timestamp columns.
- */
-public interface TimestampColumnStatistics extends ColumnStatistics {
- /**
- * Get the minimum value for the column.
- * @return minimum value
- */
- Timestamp getMinimum();
-
- /**
- * Get the maximum value for the column.
- * @return maximum value
- */
- Timestamp getMaximum();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TreeReaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TreeReaderFactory.java
index 8c13571..3fe28d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TreeReaderFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TreeReaderFactory.java
@@ -57,13 +57,25 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.orc.impl.BitFieldReader;
+import org.apache.orc.impl.DynamicByteArray;
+import org.apache.orc.impl.InStream;
+import org.apache.orc.impl.IntegerReader;
+import org.apache.orc.OrcProto;
+import org.apache.orc.impl.PositionProvider;
+import org.apache.orc.impl.RunLengthByteReader;
+import org.apache.orc.impl.RunLengthIntegerReader;
+import org.apache.orc.impl.RunLengthIntegerReaderV2;
+import org.apache.orc.impl.SerializationUtils;
+import org.apache.orc.impl.StreamName;
/**
* Factory for creating ORC tree readers.
*/
public class TreeReaderFactory {
- public static final Logger LOG = LoggerFactory.getLogger(TreeReaderFactory.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TreeReaderFactory.class);
public static class TreeReaderSchema {
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TypeDescription.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TypeDescription.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TypeDescription.java
deleted file mode 100644
index b365408..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TypeDescription.java
+++ /dev/null
@@ -1,540 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * This is the description of the types in an ORC file.
- */
-public class TypeDescription {
- private static final int MAX_PRECISION = 38;
- private static final int MAX_SCALE = 38;
- private static final int DEFAULT_PRECISION = 38;
- private static final int DEFAULT_SCALE = 10;
- private static final int DEFAULT_LENGTH = 256;
- public enum Category {
- BOOLEAN("boolean", true),
- BYTE("tinyint", true),
- SHORT("smallint", true),
- INT("int", true),
- LONG("bigint", true),
- FLOAT("float", true),
- DOUBLE("double", true),
- STRING("string", true),
- DATE("date", true),
- TIMESTAMP("timestamp", true),
- BINARY("binary", true),
- DECIMAL("decimal", true),
- VARCHAR("varchar", true),
- CHAR("char", true),
- LIST("array", false),
- MAP("map", false),
- STRUCT("struct", false),
- UNION("union", false);
-
- Category(String name, boolean isPrimitive) {
- this.name = name;
- this.isPrimitive = isPrimitive;
- }
-
- final boolean isPrimitive;
- final String name;
-
- public boolean isPrimitive() {
- return isPrimitive;
- }
-
- public String getName() {
- return name;
- }
- }
-
- public static TypeDescription createBoolean() {
- return new TypeDescription(Category.BOOLEAN);
- }
-
- public static TypeDescription createByte() {
- return new TypeDescription(Category.BYTE);
- }
-
- public static TypeDescription createShort() {
- return new TypeDescription(Category.SHORT);
- }
-
- public static TypeDescription createInt() {
- return new TypeDescription(Category.INT);
- }
-
- public static TypeDescription createLong() {
- return new TypeDescription(Category.LONG);
- }
-
- public static TypeDescription createFloat() {
- return new TypeDescription(Category.FLOAT);
- }
-
- public static TypeDescription createDouble() {
- return new TypeDescription(Category.DOUBLE);
- }
-
- public static TypeDescription createString() {
- return new TypeDescription(Category.STRING);
- }
-
- public static TypeDescription createDate() {
- return new TypeDescription(Category.DATE);
- }
-
- public static TypeDescription createTimestamp() {
- return new TypeDescription(Category.TIMESTAMP);
- }
-
- public static TypeDescription createBinary() {
- return new TypeDescription(Category.BINARY);
- }
-
- public static TypeDescription createDecimal() {
- return new TypeDescription(Category.DECIMAL);
- }
-
- /**
- * For decimal types, set the precision.
- * @param precision the new precision
- * @return this
- */
- public TypeDescription withPrecision(int precision) {
- if (category != Category.DECIMAL) {
- throw new IllegalArgumentException("precision is only allowed on decimal"+
- " and not " + category.name);
- } else if (precision < 1 || precision > MAX_PRECISION || scale > precision){
- throw new IllegalArgumentException("precision " + precision +
- " is out of range 1 .. " + scale);
- }
- this.precision = precision;
- return this;
- }
-
- /**
- * For decimal types, set the scale.
- * @param scale the new scale
- * @return this
- */
- public TypeDescription withScale(int scale) {
- if (category != Category.DECIMAL) {
- throw new IllegalArgumentException("scale is only allowed on decimal"+
- " and not " + category.name);
- } else if (scale < 0 || scale > MAX_SCALE || scale > precision) {
- throw new IllegalArgumentException("scale is out of range at " + scale);
- }
- this.scale = scale;
- return this;
- }
-
- public static TypeDescription createVarchar() {
- return new TypeDescription(Category.VARCHAR);
- }
-
- public static TypeDescription createChar() {
- return new TypeDescription(Category.CHAR);
- }
-
- /**
- * Set the maximum length for char and varchar types.
- * @param maxLength the maximum value
- * @return this
- */
- public TypeDescription withMaxLength(int maxLength) {
- if (category != Category.VARCHAR && category != Category.CHAR) {
- throw new IllegalArgumentException("maxLength is only allowed on char" +
- " and varchar and not " + category.name);
- }
- this.maxLength = maxLength;
- return this;
- }
-
- public static TypeDescription createList(TypeDescription childType) {
- TypeDescription result = new TypeDescription(Category.LIST);
- result.children.add(childType);
- childType.parent = result;
- return result;
- }
-
- public static TypeDescription createMap(TypeDescription keyType,
- TypeDescription valueType) {
- TypeDescription result = new TypeDescription(Category.MAP);
- result.children.add(keyType);
- result.children.add(valueType);
- keyType.parent = result;
- valueType.parent = result;
- return result;
- }
-
- public static TypeDescription createUnion() {
- return new TypeDescription(Category.UNION);
- }
-
- public static TypeDescription createStruct() {
- return new TypeDescription(Category.STRUCT);
- }
-
- /**
- * Add a child to a union type.
- * @param child a new child type to add
- * @return the union type.
- */
- public TypeDescription addUnionChild(TypeDescription child) {
- if (category != Category.UNION) {
- throw new IllegalArgumentException("Can only add types to union type" +
- " and not " + category);
- }
- children.add(child);
- child.parent = this;
- return this;
- }
-
- /**
- * Add a field to a struct type as it is built.
- * @param field the field name
- * @param fieldType the type of the field
- * @return the struct type
- */
- public TypeDescription addField(String field, TypeDescription fieldType) {
- if (category != Category.STRUCT) {
- throw new IllegalArgumentException("Can only add fields to struct type" +
- " and not " + category);
- }
- fieldNames.add(field);
- children.add(fieldType);
- fieldType.parent = this;
- return this;
- }
-
- /**
- * Get the id for this type.
- * The first call will cause all of the the ids in tree to be assigned, so
- * it should not be called before the type is completely built.
- * @return the sequential id
- */
- public int getId() {
- // if the id hasn't been assigned, assign all of the ids from the root
- if (id == -1) {
- TypeDescription root = this;
- while (root.parent != null) {
- root = root.parent;
- }
- root.assignIds(0);
- }
- return id;
- }
-
- /**
- * Get the maximum id assigned to this type or its children.
- * The first call will cause all of the the ids in tree to be assigned, so
- * it should not be called before the type is completely built.
- * @return the maximum id assigned under this type
- */
- public int getMaximumId() {
- // if the id hasn't been assigned, assign all of the ids from the root
- if (maxId == -1) {
- TypeDescription root = this;
- while (root.parent != null) {
- root = root.parent;
- }
- root.assignIds(0);
- }
- return maxId;
- }
-
- private ColumnVector createColumn() {
- switch (category) {
- case BOOLEAN:
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- case TIMESTAMP:
- case DATE:
- return new LongColumnVector();
- case FLOAT:
- case DOUBLE:
- return new DoubleColumnVector();
- case DECIMAL:
- return new DecimalColumnVector(precision, scale);
- case STRING:
- case BINARY:
- case CHAR:
- case VARCHAR:
- return new BytesColumnVector();
- case STRUCT: {
- ColumnVector[] fieldVector = new ColumnVector[children.size()];
- for(int i=0; i < fieldVector.length; ++i) {
- fieldVector[i] = children.get(i).createColumn();
- }
- return new StructColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
- fieldVector);
- }
- case UNION: {
- ColumnVector[] fieldVector = new ColumnVector[children.size()];
- for(int i=0; i < fieldVector.length; ++i) {
- fieldVector[i] = children.get(i).createColumn();
- }
- return new UnionColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
- fieldVector);
- }
- case LIST:
- return new ListColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
- children.get(0).createColumn());
- case MAP:
- return new MapColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
- children.get(0).createColumn(), children.get(1).createColumn());
- default:
- throw new IllegalArgumentException("Unknown type " + category);
- }
- }
-
- public VectorizedRowBatch createRowBatch() {
- VectorizedRowBatch result;
- if (category == Category.STRUCT) {
- result = new VectorizedRowBatch(children.size(),
- VectorizedRowBatch.DEFAULT_SIZE);
- for(int i=0; i < result.cols.length; ++i) {
- result.cols[i] = children.get(i).createColumn();
- }
- } else {
- result = new VectorizedRowBatch(1, VectorizedRowBatch.DEFAULT_SIZE);
- result.cols[0] = createColumn();
- }
- result.reset();
- return result;
- }
-
- /**
- * Get the kind of this type.
- * @return get the category for this type.
- */
- public Category getCategory() {
- return category;
- }
-
- /**
- * Get the maximum length of the type. Only used for char and varchar types.
- * @return the maximum length of the string type
- */
- public int getMaxLength() {
- return maxLength;
- }
-
- /**
- * Get the precision of the decimal type.
- * @return the number of digits for the precision.
- */
- public int getPrecision() {
- return precision;
- }
-
- /**
- * Get the scale of the decimal type.
- * @return the number of digits for the scale.
- */
- public int getScale() {
- return scale;
- }
-
- /**
- * For struct types, get the list of field names.
- * @return the list of field names.
- */
- public List<String> getFieldNames() {
- return Collections.unmodifiableList(fieldNames);
- }
-
- /**
- * Get the subtypes of this type.
- * @return the list of children types
- */
- public List<TypeDescription> getChildren() {
- return children == null ? null : Collections.unmodifiableList(children);
- }
-
- /**
- * Assign ids to all of the nodes under this one.
- * @param startId the lowest id to assign
- * @return the next available id
- */
- private int assignIds(int startId) {
- id = startId++;
- if (children != null) {
- for (TypeDescription child : children) {
- startId = child.assignIds(startId);
- }
- }
- maxId = startId - 1;
- return startId;
- }
-
- private TypeDescription(Category category) {
- this.category = category;
- if (category.isPrimitive) {
- children = null;
- } else {
- children = new ArrayList<>();
- }
- if (category == Category.STRUCT) {
- fieldNames = new ArrayList<>();
- } else {
- fieldNames = null;
- }
- }
-
- private int id = -1;
- private int maxId = -1;
- private TypeDescription parent;
- private final Category category;
- private final List<TypeDescription> children;
- private final List<String> fieldNames;
- private int maxLength = DEFAULT_LENGTH;
- private int precision = DEFAULT_PRECISION;
- private int scale = DEFAULT_SCALE;
-
- public void printToBuffer(StringBuilder buffer) {
- buffer.append(category.name);
- switch (category) {
- case DECIMAL:
- buffer.append('(');
- buffer.append(precision);
- buffer.append(',');
- buffer.append(scale);
- buffer.append(')');
- break;
- case CHAR:
- case VARCHAR:
- buffer.append('(');
- buffer.append(maxLength);
- buffer.append(')');
- break;
- case LIST:
- case MAP:
- case UNION:
- buffer.append('<');
- for(int i=0; i < children.size(); ++i) {
- if (i != 0) {
- buffer.append(',');
- }
- children.get(i).printToBuffer(buffer);
- }
- buffer.append('>');
- break;
- case STRUCT:
- buffer.append('<');
- for(int i=0; i < children.size(); ++i) {
- if (i != 0) {
- buffer.append(',');
- }
- buffer.append(fieldNames.get(i));
- buffer.append(':');
- children.get(i).printToBuffer(buffer);
- }
- buffer.append('>');
- break;
- default:
- break;
- }
- }
-
- public String toString() {
- StringBuilder buffer = new StringBuilder();
- printToBuffer(buffer);
- return buffer.toString();
- }
-
- private void printJsonToBuffer(String prefix, StringBuilder buffer,
- int indent) {
- for(int i=0; i < indent; ++i) {
- buffer.append(' ');
- }
- buffer.append(prefix);
- buffer.append("{\"category\": \"");
- buffer.append(category.name);
- buffer.append("\", \"id\": ");
- buffer.append(getId());
- buffer.append(", \"max\": ");
- buffer.append(maxId);
- switch (category) {
- case DECIMAL:
- buffer.append(", \"precision\": ");
- buffer.append(precision);
- buffer.append(", \"scale\": ");
- buffer.append(scale);
- break;
- case CHAR:
- case VARCHAR:
- buffer.append(", \"length\": ");
- buffer.append(maxLength);
- break;
- case LIST:
- case MAP:
- case UNION:
- buffer.append(", \"children\": [");
- for(int i=0; i < children.size(); ++i) {
- buffer.append('\n');
- children.get(i).printJsonToBuffer("", buffer, indent + 2);
- if (i != children.size() - 1) {
- buffer.append(',');
- }
- }
- buffer.append("]");
- break;
- case STRUCT:
- buffer.append(", \"fields\": [");
- for(int i=0; i < children.size(); ++i) {
- buffer.append('\n');
- children.get(i).printJsonToBuffer("\"" + fieldNames.get(i) + "\": ",
- buffer, indent + 2);
- if (i != children.size() - 1) {
- buffer.append(',');
- }
- }
- buffer.append(']');
- break;
- default:
- break;
- }
- buffer.append('}');
- }
-
- public String toJson() {
- StringBuilder buffer = new StringBuilder();
- printJsonToBuffer("", buffer, 0);
- return buffer.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
index 6d280c8..e08aaf3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.orc.OrcProto;
+import org.apache.orc.TypeDescription;
/**
* A MapReduce/Hive input format for ORC files.
@@ -64,7 +66,7 @@ public class VectorizedOrcInputFormat extends FileInputFormat<NullWritable, Vect
/**
* Do we have schema on read in the configuration variables?
*/
- TypeDescription schema = OrcUtils.getDesiredRowTypeDescr(conf);
+ TypeDescription schema = OrcInputFormat.getDesiredRowTypeDescr(conf);
List<OrcProto.Type> types = file.getTypes();
Reader.Options options = new Reader.Options();
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java
index 1873ed1..1f5927a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java
@@ -18,32 +18,12 @@
package org.apache.hadoop.hive.ql.io.orc;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
/**
- * The interface for writing ORC files.
+ * The HIVE interface for writing ORC files.
*/
-public interface Writer {
-
- /**
- * Get the schema for this writer
- * @return the file schema
- */
- TypeDescription getSchema();
-
- /**
- * Add arbitrary meta-data to the ORC file. This may be called at any point
- * until the Writer is closed. If the same key is passed a second time, the
- * second value will replace the first.
- * @param key a key to label the data with.
- * @param value the contents of the metadata.
- */
- void addUserMetadata(String key, ByteBuffer value);
+public interface Writer extends org.apache.orc.Writer {
/**
* Add a row to the ORC file.
@@ -51,68 +31,4 @@ public interface Writer {
* @throws IOException
*/
void addRow(Object row) throws IOException;
-
- /**
- * Add a row batch to the ORC file.
- * @param batch the rows to add
- */
- void addRowBatch(VectorizedRowBatch batch) throws IOException;
-
- /**
- * Flush all of the buffers and close the file. No methods on this writer
- * should be called afterwards.
- * @throws IOException
- */
- void close() throws IOException;
-
- /**
- * Return the deserialized data size. Raw data size will be compute when
- * writing the file footer. Hence raw data size value will be available only
- * after closing the writer.
- *
- * @return raw data size
- */
- long getRawDataSize();
-
- /**
- * Return the number of rows in file. Row count gets updated when flushing
- * the stripes. To get accurate row count this method should be called after
- * closing the writer.
- *
- * @return row count
- */
- long getNumberOfRows();
-
- /**
- * Write an intermediate footer on the file such that if the file is
- * truncated to the returned offset, it would be a valid ORC file.
- * @return the offset that would be a valid end location for an ORC file
- */
- long writeIntermediateFooter() throws IOException;
-
- /**
- * Fast stripe append to ORC file. This interface is used for fast ORC file
- * merge with other ORC files. When merging, the file to be merged should pass
- * stripe in binary form along with stripe information and stripe statistics.
- * After appending last stripe of a file, use appendUserMetadata() to append
- * any user metadata.
- * @param stripe - stripe as byte array
- * @param offset - offset within byte array
- * @param length - length of stripe within byte array
- * @param stripeInfo - stripe information
- * @param stripeStatistics - stripe statistics (Protobuf objects can be
- * merged directly)
- * @throws IOException
- */
- public void appendStripe(byte[] stripe, int offset, int length,
- StripeInformation stripeInfo,
- OrcProto.StripeStatistics stripeStatistics) throws IOException;
-
- /**
- * When fast stripe append is used for merging ORC stripes, after appending
- * the last stripe from a file, this interface must be used to merge any
- * user metadata.
- * @param userMetadata - user metadata
- */
- public void appendUserMetadata(List<OrcProto.UserMetadataItem> userMetadata);
}
[13/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/TypeDescription.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/TypeDescription.java b/orc/src/java/org/apache/orc/TypeDescription.java
new file mode 100644
index 0000000..fc945e4
--- /dev/null
+++ b/orc/src/java/org/apache/orc/TypeDescription.java
@@ -0,0 +1,540 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * This is the description of the types in an ORC file.
+ */
+public class TypeDescription {
+ private static final int MAX_PRECISION = 38;
+ private static final int MAX_SCALE = 38;
+ private static final int DEFAULT_PRECISION = 38;
+ private static final int DEFAULT_SCALE = 10;
+ private static final int DEFAULT_LENGTH = 256;
+ public enum Category {
+ BOOLEAN("boolean", true),
+ BYTE("tinyint", true),
+ SHORT("smallint", true),
+ INT("int", true),
+ LONG("bigint", true),
+ FLOAT("float", true),
+ DOUBLE("double", true),
+ STRING("string", true),
+ DATE("date", true),
+ TIMESTAMP("timestamp", true),
+ BINARY("binary", true),
+ DECIMAL("decimal", true),
+ VARCHAR("varchar", true),
+ CHAR("char", true),
+ LIST("array", false),
+ MAP("map", false),
+ STRUCT("struct", false),
+ UNION("union", false);
+
+ Category(String name, boolean isPrimitive) {
+ this.name = name;
+ this.isPrimitive = isPrimitive;
+ }
+
+ final boolean isPrimitive;
+ final String name;
+
+ public boolean isPrimitive() {
+ return isPrimitive;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+
+ public static TypeDescription createBoolean() {
+ return new TypeDescription(Category.BOOLEAN);
+ }
+
+ public static TypeDescription createByte() {
+ return new TypeDescription(Category.BYTE);
+ }
+
+ public static TypeDescription createShort() {
+ return new TypeDescription(Category.SHORT);
+ }
+
+ public static TypeDescription createInt() {
+ return new TypeDescription(Category.INT);
+ }
+
+ public static TypeDescription createLong() {
+ return new TypeDescription(Category.LONG);
+ }
+
+ public static TypeDescription createFloat() {
+ return new TypeDescription(Category.FLOAT);
+ }
+
+ public static TypeDescription createDouble() {
+ return new TypeDescription(Category.DOUBLE);
+ }
+
+ public static TypeDescription createString() {
+ return new TypeDescription(Category.STRING);
+ }
+
+ public static TypeDescription createDate() {
+ return new TypeDescription(Category.DATE);
+ }
+
+ public static TypeDescription createTimestamp() {
+ return new TypeDescription(Category.TIMESTAMP);
+ }
+
+ public static TypeDescription createBinary() {
+ return new TypeDescription(Category.BINARY);
+ }
+
+ public static TypeDescription createDecimal() {
+ return new TypeDescription(Category.DECIMAL);
+ }
+
+ /**
+ * For decimal types, set the precision.
+ * @param precision the new precision
+ * @return this
+ */
+ public TypeDescription withPrecision(int precision) {
+ if (category != Category.DECIMAL) {
+ throw new IllegalArgumentException("precision is only allowed on decimal"+
+ " and not " + category.name);
+ } else if (precision < 1 || precision > MAX_PRECISION || scale > precision){
+ throw new IllegalArgumentException("precision " + precision +
+ " is out of range 1 .. " + scale);
+ }
+ this.precision = precision;
+ return this;
+ }
+
+ /**
+ * For decimal types, set the scale.
+ * @param scale the new scale
+ * @return this
+ */
+ public TypeDescription withScale(int scale) {
+ if (category != Category.DECIMAL) {
+ throw new IllegalArgumentException("scale is only allowed on decimal"+
+ " and not " + category.name);
+ } else if (scale < 0 || scale > MAX_SCALE || scale > precision) {
+ throw new IllegalArgumentException("scale is out of range at " + scale);
+ }
+ this.scale = scale;
+ return this;
+ }
+
+ public static TypeDescription createVarchar() {
+ return new TypeDescription(Category.VARCHAR);
+ }
+
+ public static TypeDescription createChar() {
+ return new TypeDescription(Category.CHAR);
+ }
+
+ /**
+ * Set the maximum length for char and varchar types.
+ * @param maxLength the maximum value
+ * @return this
+ */
+ public TypeDescription withMaxLength(int maxLength) {
+ if (category != Category.VARCHAR && category != Category.CHAR) {
+ throw new IllegalArgumentException("maxLength is only allowed on char" +
+ " and varchar and not " + category.name);
+ }
+ this.maxLength = maxLength;
+ return this;
+ }
+
+ public static TypeDescription createList(TypeDescription childType) {
+ TypeDescription result = new TypeDescription(Category.LIST);
+ result.children.add(childType);
+ childType.parent = result;
+ return result;
+ }
+
+ public static TypeDescription createMap(TypeDescription keyType,
+ TypeDescription valueType) {
+ TypeDescription result = new TypeDescription(Category.MAP);
+ result.children.add(keyType);
+ result.children.add(valueType);
+ keyType.parent = result;
+ valueType.parent = result;
+ return result;
+ }
+
+ public static TypeDescription createUnion() {
+ return new TypeDescription(Category.UNION);
+ }
+
+ public static TypeDescription createStruct() {
+ return new TypeDescription(Category.STRUCT);
+ }
+
+ /**
+ * Add a child to a union type.
+ * @param child a new child type to add
+ * @return the union type.
+ */
+ public TypeDescription addUnionChild(TypeDescription child) {
+ if (category != Category.UNION) {
+ throw new IllegalArgumentException("Can only add types to union type" +
+ " and not " + category);
+ }
+ children.add(child);
+ child.parent = this;
+ return this;
+ }
+
+ /**
+ * Add a field to a struct type as it is built.
+ * @param field the field name
+ * @param fieldType the type of the field
+ * @return the struct type
+ */
+ public TypeDescription addField(String field, TypeDescription fieldType) {
+ if (category != Category.STRUCT) {
+ throw new IllegalArgumentException("Can only add fields to struct type" +
+ " and not " + category);
+ }
+ fieldNames.add(field);
+ children.add(fieldType);
+ fieldType.parent = this;
+ return this;
+ }
+
+ /**
+ * Get the id for this type.
+ * The first call will cause all of the the ids in tree to be assigned, so
+ * it should not be called before the type is completely built.
+ * @return the sequential id
+ */
+ public int getId() {
+ // if the id hasn't been assigned, assign all of the ids from the root
+ if (id == -1) {
+ TypeDescription root = this;
+ while (root.parent != null) {
+ root = root.parent;
+ }
+ root.assignIds(0);
+ }
+ return id;
+ }
+
+ /**
+ * Get the maximum id assigned to this type or its children.
+ * The first call will cause all of the the ids in tree to be assigned, so
+ * it should not be called before the type is completely built.
+ * @return the maximum id assigned under this type
+ */
+ public int getMaximumId() {
+ // if the id hasn't been assigned, assign all of the ids from the root
+ if (maxId == -1) {
+ TypeDescription root = this;
+ while (root.parent != null) {
+ root = root.parent;
+ }
+ root.assignIds(0);
+ }
+ return maxId;
+ }
+
+ private ColumnVector createColumn() {
+ switch (category) {
+ case BOOLEAN:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ case TIMESTAMP:
+ case DATE:
+ return new LongColumnVector();
+ case FLOAT:
+ case DOUBLE:
+ return new DoubleColumnVector();
+ case DECIMAL:
+ return new DecimalColumnVector(precision, scale);
+ case STRING:
+ case BINARY:
+ case CHAR:
+ case VARCHAR:
+ return new BytesColumnVector();
+ case STRUCT: {
+ ColumnVector[] fieldVector = new ColumnVector[children.size()];
+ for(int i=0; i < fieldVector.length; ++i) {
+ fieldVector[i] = children.get(i).createColumn();
+ }
+ return new StructColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
+ fieldVector);
+ }
+ case UNION: {
+ ColumnVector[] fieldVector = new ColumnVector[children.size()];
+ for(int i=0; i < fieldVector.length; ++i) {
+ fieldVector[i] = children.get(i).createColumn();
+ }
+ return new UnionColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
+ fieldVector);
+ }
+ case LIST:
+ return new ListColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
+ children.get(0).createColumn());
+ case MAP:
+ return new MapColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
+ children.get(0).createColumn(), children.get(1).createColumn());
+ default:
+ throw new IllegalArgumentException("Unknown type " + category);
+ }
+ }
+
+ public VectorizedRowBatch createRowBatch() {
+ VectorizedRowBatch result;
+ if (category == Category.STRUCT) {
+ result = new VectorizedRowBatch(children.size(),
+ VectorizedRowBatch.DEFAULT_SIZE);
+ for(int i=0; i < result.cols.length; ++i) {
+ result.cols[i] = children.get(i).createColumn();
+ }
+ } else {
+ result = new VectorizedRowBatch(1, VectorizedRowBatch.DEFAULT_SIZE);
+ result.cols[0] = createColumn();
+ }
+ result.reset();
+ return result;
+ }
+
+ /**
+ * Get the kind of this type.
+ * @return get the category for this type.
+ */
+ public Category getCategory() {
+ return category;
+ }
+
+ /**
+ * Get the maximum length of the type. Only used for char and varchar types.
+ * @return the maximum length of the string type
+ */
+ public int getMaxLength() {
+ return maxLength;
+ }
+
+ /**
+ * Get the precision of the decimal type.
+ * @return the number of digits for the precision.
+ */
+ public int getPrecision() {
+ return precision;
+ }
+
+ /**
+ * Get the scale of the decimal type.
+ * @return the number of digits for the scale.
+ */
+ public int getScale() {
+ return scale;
+ }
+
+ /**
+ * For struct types, get the list of field names.
+ * @return the list of field names.
+ */
+ public List<String> getFieldNames() {
+ return Collections.unmodifiableList(fieldNames);
+ }
+
+ /**
+ * Get the subtypes of this type.
+ * @return the list of children types
+ */
+ public List<TypeDescription> getChildren() {
+ return children == null ? null : Collections.unmodifiableList(children);
+ }
+
+ /**
+ * Assign ids to all of the nodes under this one.
+ * @param startId the lowest id to assign
+ * @return the next available id
+ */
+ private int assignIds(int startId) {
+ id = startId++;
+ if (children != null) {
+ for (TypeDescription child : children) {
+ startId = child.assignIds(startId);
+ }
+ }
+ maxId = startId - 1;
+ return startId;
+ }
+
+ private TypeDescription(Category category) {
+ this.category = category;
+ if (category.isPrimitive) {
+ children = null;
+ } else {
+ children = new ArrayList<>();
+ }
+ if (category == Category.STRUCT) {
+ fieldNames = new ArrayList<>();
+ } else {
+ fieldNames = null;
+ }
+ }
+
+ private int id = -1;
+ private int maxId = -1;
+ private TypeDescription parent;
+ private final Category category;
+ private final List<TypeDescription> children;
+ private final List<String> fieldNames;
+ private int maxLength = DEFAULT_LENGTH;
+ private int precision = DEFAULT_PRECISION;
+ private int scale = DEFAULT_SCALE;
+
+ public void printToBuffer(StringBuilder buffer) {
+ buffer.append(category.name);
+ switch (category) {
+ case DECIMAL:
+ buffer.append('(');
+ buffer.append(precision);
+ buffer.append(',');
+ buffer.append(scale);
+ buffer.append(')');
+ break;
+ case CHAR:
+ case VARCHAR:
+ buffer.append('(');
+ buffer.append(maxLength);
+ buffer.append(')');
+ break;
+ case LIST:
+ case MAP:
+ case UNION:
+ buffer.append('<');
+ for(int i=0; i < children.size(); ++i) {
+ if (i != 0) {
+ buffer.append(',');
+ }
+ children.get(i).printToBuffer(buffer);
+ }
+ buffer.append('>');
+ break;
+ case STRUCT:
+ buffer.append('<');
+ for(int i=0; i < children.size(); ++i) {
+ if (i != 0) {
+ buffer.append(',');
+ }
+ buffer.append(fieldNames.get(i));
+ buffer.append(':');
+ children.get(i).printToBuffer(buffer);
+ }
+ buffer.append('>');
+ break;
+ default:
+ break;
+ }
+ }
+
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ printToBuffer(buffer);
+ return buffer.toString();
+ }
+
+ private void printJsonToBuffer(String prefix, StringBuilder buffer,
+ int indent) {
+ for(int i=0; i < indent; ++i) {
+ buffer.append(' ');
+ }
+ buffer.append(prefix);
+ buffer.append("{\"category\": \"");
+ buffer.append(category.name);
+ buffer.append("\", \"id\": ");
+ buffer.append(getId());
+ buffer.append(", \"max\": ");
+ buffer.append(maxId);
+ switch (category) {
+ case DECIMAL:
+ buffer.append(", \"precision\": ");
+ buffer.append(precision);
+ buffer.append(", \"scale\": ");
+ buffer.append(scale);
+ break;
+ case CHAR:
+ case VARCHAR:
+ buffer.append(", \"length\": ");
+ buffer.append(maxLength);
+ break;
+ case LIST:
+ case MAP:
+ case UNION:
+ buffer.append(", \"children\": [");
+ for(int i=0; i < children.size(); ++i) {
+ buffer.append('\n');
+ children.get(i).printJsonToBuffer("", buffer, indent + 2);
+ if (i != children.size() - 1) {
+ buffer.append(',');
+ }
+ }
+ buffer.append("]");
+ break;
+ case STRUCT:
+ buffer.append(", \"fields\": [");
+ for(int i=0; i < children.size(); ++i) {
+ buffer.append('\n');
+ children.get(i).printJsonToBuffer("\"" + fieldNames.get(i) + "\": ",
+ buffer, indent + 2);
+ if (i != children.size() - 1) {
+ buffer.append(',');
+ }
+ }
+ buffer.append(']');
+ break;
+ default:
+ break;
+ }
+ buffer.append('}');
+ }
+
+ public String toJson() {
+ StringBuilder buffer = new StringBuilder();
+ printJsonToBuffer("", buffer, 0);
+ return buffer.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/Writer.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/Writer.java b/orc/src/java/org/apache/orc/Writer.java
new file mode 100644
index 0000000..4492062
--- /dev/null
+++ b/orc/src/java/org/apache/orc/Writer.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.orc.OrcProto;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.TypeDescription;
+
+/**
+ * The interface for writing ORC files.
+ */
+public interface Writer {
+
+ /**
+ * Get the schema for this writer
+ * @return the file schema
+ */
+ TypeDescription getSchema();
+
+ /**
+ * Add arbitrary meta-data to the ORC file. This may be called at any point
+ * until the Writer is closed. If the same key is passed a second time, the
+ * second value will replace the first.
+ * @param key a key to label the data with.
+ * @param value the contents of the metadata.
+ */
+ void addUserMetadata(String key, ByteBuffer value);
+
+ /**
+ * Add a row batch to the ORC file.
+ * @param batch the rows to add
+ */
+ void addRowBatch(VectorizedRowBatch batch) throws IOException;
+
+ /**
+ * Flush all of the buffers and close the file. No methods on this writer
+ * should be called afterwards.
+ * @throws IOException
+ */
+ void close() throws IOException;
+
+ /**
+ * Return the deserialized data size. Raw data size will be compute when
+ * writing the file footer. Hence raw data size value will be available only
+ * after closing the writer.
+ *
+ * @return raw data size
+ */
+ long getRawDataSize();
+
+ /**
+ * Return the number of rows in file. Row count gets updated when flushing
+ * the stripes. To get accurate row count this method should be called after
+ * closing the writer.
+ *
+ * @return row count
+ */
+ long getNumberOfRows();
+
+ /**
+ * Write an intermediate footer on the file such that if the file is
+ * truncated to the returned offset, it would be a valid ORC file.
+ * @return the offset that would be a valid end location for an ORC file
+ */
+ long writeIntermediateFooter() throws IOException;
+
+ /**
+ * Fast stripe append to ORC file. This interface is used for fast ORC file
+ * merge with other ORC files. When merging, the file to be merged should pass
+ * stripe in binary form along with stripe information and stripe statistics.
+ * After appending last stripe of a file, use appendUserMetadata() to append
+ * any user metadata.
+ * @param stripe - stripe as byte array
+ * @param offset - offset within byte array
+ * @param length - length of stripe within byte array
+ * @param stripeInfo - stripe information
+ * @param stripeStatistics - stripe statistics (Protobuf objects can be
+ * merged directly)
+ * @throws IOException
+ */
+ public void appendStripe(byte[] stripe, int offset, int length,
+ StripeInformation stripeInfo,
+ OrcProto.StripeStatistics stripeStatistics) throws IOException;
+
+ /**
+ * When fast stripe append is used for merging ORC stripes, after appending
+ * the last stripe from a file, this interface must be used to merge any
+ * user metadata.
+ * @param userMetadata - user metadata
+ */
+ public void appendUserMetadata(List<OrcProto.UserMetadataItem> userMetadata);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/BitFieldReader.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/BitFieldReader.java b/orc/src/java/org/apache/orc/impl/BitFieldReader.java
new file mode 100644
index 0000000..8d9d3cb
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/BitFieldReader.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.orc.impl.InStream;
+import org.apache.orc.impl.PositionProvider;
+import org.apache.orc.impl.RunLengthByteReader;
+
+public class BitFieldReader {
+ private final RunLengthByteReader input;
+ /** The number of bits in one item. Non-test code always uses 1. */
+ private final int bitSize;
+ private int current;
+ private int bitsLeft;
+ private final int mask;
+
+ public BitFieldReader(InStream input,
+ int bitSize) throws IOException {
+ this.input = new RunLengthByteReader(input);
+ this.bitSize = bitSize;
+ mask = (1 << bitSize) - 1;
+ }
+
+ public void setInStream(InStream inStream) {
+ this.input.setInStream(inStream);
+ }
+
+ private void readByte() throws IOException {
+ if (input.hasNext()) {
+ current = 0xff & input.next();
+ bitsLeft = 8;
+ } else {
+ throw new EOFException("Read past end of bit field from " + this);
+ }
+ }
+
+ public int next() throws IOException {
+ int result = 0;
+ int bitsLeftToRead = bitSize;
+ while (bitsLeftToRead > bitsLeft) {
+ result <<= bitsLeft;
+ result |= current & ((1 << bitsLeft) - 1);
+ bitsLeftToRead -= bitsLeft;
+ readByte();
+ }
+ if (bitsLeftToRead > 0) {
+ result <<= bitsLeftToRead;
+ bitsLeft -= bitsLeftToRead;
+ result |= (current >>> bitsLeft) & ((1 << bitsLeftToRead) - 1);
+ }
+ return result & mask;
+ }
+
+ /**
+ * Unlike integer readers, where runs are encoded explicitly, in this one we have to read ahead
+ * to figure out whether we have a run. Given that runs in booleans are likely it's worth it.
+ * However it means we'd need to keep track of how many bytes we read, and next/nextVector won't
+ * work anymore once this is called. These is trivial to fix, but these are never interspersed.
+ */
+ private boolean lastRunValue;
+ private int lastRunLength = -1;
+ private void readNextRun(int maxRunLength) throws IOException {
+ assert bitSize == 1;
+ if (lastRunLength > 0) return; // last run is not exhausted yet
+ if (bitsLeft == 0) {
+ readByte();
+ }
+ // First take care of the partial bits.
+ boolean hasVal = false;
+ int runLength = 0;
+ if (bitsLeft != 8) {
+ int partialBitsMask = (1 << bitsLeft) - 1;
+ int partialBits = current & partialBitsMask;
+ if (partialBits == partialBitsMask || partialBits == 0) {
+ lastRunValue = (partialBits == partialBitsMask);
+ if (maxRunLength <= bitsLeft) {
+ lastRunLength = maxRunLength;
+ return;
+ }
+ maxRunLength -= bitsLeft;
+ hasVal = true;
+ runLength = bitsLeft;
+ bitsLeft = 0;
+ } else {
+ // There's no run in partial bits. Return whatever we have.
+ int prefixBitsCount = 32 - bitsLeft;
+ runLength = Integer.numberOfLeadingZeros(partialBits) - prefixBitsCount;
+ lastRunValue = (runLength > 0);
+ lastRunLength = Math.min(maxRunLength, lastRunValue ? runLength :
+ (Integer.numberOfLeadingZeros(~(partialBits | ~partialBitsMask)) - prefixBitsCount));
+ return;
+ }
+ assert bitsLeft == 0;
+ readByte();
+ }
+ if (!hasVal) {
+ lastRunValue = ((current >> 7) == 1);
+ hasVal = true;
+ }
+ // Read full bytes until the run ends.
+ assert bitsLeft == 8;
+ while (maxRunLength >= 8
+ && ((lastRunValue && (current == 0xff)) || (!lastRunValue && (current == 0)))) {
+ runLength += 8;
+ maxRunLength -= 8;
+ readByte();
+ }
+ if (maxRunLength > 0) {
+ int extraBits = Integer.numberOfLeadingZeros(
+ lastRunValue ? (~(current | ~255)) : current) - 24;
+ bitsLeft -= extraBits;
+ runLength += extraBits;
+ }
+ lastRunLength = runLength;
+ }
+
+ public void nextVector(LongColumnVector previous,
+ long previousLen) throws IOException {
+ previous.isRepeating = true;
+ for (int i = 0; i < previousLen; i++) {
+ if (!previous.isNull[i]) {
+ previous.vector[i] = next();
+ } else {
+ // The default value of null for int types in vectorized
+ // processing is 1, so set that if the value is null
+ previous.vector[i] = 1;
+ }
+
+ // The default value for nulls in Vectorization for int types is 1
+ // and given that non null value can also be 1, we need to check for isNull also
+ // when determining the isRepeating flag.
+ if (previous.isRepeating
+ && i > 0
+ && ((previous.vector[i - 1] != previous.vector[i]) || (previous.isNull[i - 1] != previous.isNull[i]))) {
+ previous.isRepeating = false;
+ }
+ }
+ }
+
+ public void seek(PositionProvider index) throws IOException {
+ input.seek(index);
+ int consumed = (int) index.getNext();
+ if (consumed > 8) {
+ throw new IllegalArgumentException("Seek past end of byte at " +
+ consumed + " in " + input);
+ } else if (consumed != 0) {
+ readByte();
+ bitsLeft = 8 - consumed;
+ } else {
+ bitsLeft = 0;
+ }
+ }
+
+ public void skip(long items) throws IOException {
+ long totalBits = bitSize * items;
+ if (bitsLeft >= totalBits) {
+ bitsLeft -= totalBits;
+ } else {
+ totalBits -= bitsLeft;
+ input.skip(totalBits / 8);
+ current = input.next();
+ bitsLeft = (int) (8 - (totalBits % 8));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "bit reader current: " + current + " bits left: " + bitsLeft +
+ " bit size: " + bitSize + " from " + input;
+ }
+
+ boolean hasFullByte() {
+ return bitsLeft == 8 || bitsLeft == 0;
+ }
+
+ int peekOneBit() throws IOException {
+ assert bitSize == 1;
+ if (bitsLeft == 0) {
+ readByte();
+ }
+ return (current >>> (bitsLeft - 1)) & 1;
+ }
+
+ int peekFullByte() throws IOException {
+ assert bitSize == 1;
+ assert bitsLeft == 8 || bitsLeft == 0;
+ if (bitsLeft == 0) {
+ readByte();
+ }
+ return current;
+ }
+
+ void skipInCurrentByte(int bits) throws IOException {
+ assert bitsLeft >= bits;
+ bitsLeft -= bits;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/BitFieldWriter.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/BitFieldWriter.java b/orc/src/java/org/apache/orc/impl/BitFieldWriter.java
new file mode 100644
index 0000000..aa5f886
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/BitFieldWriter.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import org.apache.orc.impl.PositionRecorder;
+import org.apache.orc.impl.PositionedOutputStream;
+import org.apache.orc.impl.RunLengthByteWriter;
+
+import java.io.IOException;
+
+public class BitFieldWriter {
+ private RunLengthByteWriter output;
+ private final int bitSize;
+ private byte current = 0;
+ private int bitsLeft = 8;
+
+ public BitFieldWriter(PositionedOutputStream output,
+ int bitSize) throws IOException {
+ this.output = new RunLengthByteWriter(output);
+ this.bitSize = bitSize;
+ }
+
+ private void writeByte() throws IOException {
+ output.write(current);
+ current = 0;
+ bitsLeft = 8;
+ }
+
+ public void flush() throws IOException {
+ if (bitsLeft != 8) {
+ writeByte();
+ }
+ output.flush();
+ }
+
+ public void write(int value) throws IOException {
+ int bitsToWrite = bitSize;
+ while (bitsToWrite > bitsLeft) {
+ // add the bits to the bottom of the current word
+ current |= value >>> (bitsToWrite - bitsLeft);
+ // subtract out the bits we just added
+ bitsToWrite -= bitsLeft;
+ // zero out the bits above bitsToWrite
+ value &= (1 << bitsToWrite) - 1;
+ writeByte();
+ }
+ bitsLeft -= bitsToWrite;
+ current |= value << bitsLeft;
+ if (bitsLeft == 0) {
+ writeByte();
+ }
+ }
+
+ public void getPosition(PositionRecorder recorder) throws IOException {
+ output.getPosition(recorder);
+ recorder.addPosition(8 - bitsLeft);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/BufferChunk.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/BufferChunk.java b/orc/src/java/org/apache/orc/impl/BufferChunk.java
new file mode 100644
index 0000000..da43b96
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/BufferChunk.java
@@ -0,0 +1,85 @@
+package org.apache.orc.impl;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+
+/**
+ * The sections of stripe that we have read.
+ * This might not match diskRange - 1 disk range can be multiple buffer chunks,
+ * depending on DFS block boundaries.
+ */
+public class BufferChunk extends DiskRangeList {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BufferChunk.class);
+ final ByteBuffer chunk;
+
+ public BufferChunk(ByteBuffer chunk, long offset) {
+ super(offset, offset + chunk.remaining());
+ this.chunk = chunk;
+ }
+
+ public ByteBuffer getChunk() {
+ return chunk;
+ }
+
+ @Override
+ public boolean hasData() {
+ return chunk != null;
+ }
+
+ @Override
+ public final String toString() {
+ boolean makesSense = chunk.remaining() == (end - offset);
+ return "data range [" + offset + ", " + end + "), size: " + chunk.remaining()
+ + (makesSense ? "" : "(!)") + " type: " +
+ (chunk.isDirect() ? "direct" : "array-backed");
+ }
+
+ @Override
+ public DiskRange sliceAndShift(long offset, long end, long shiftBy) {
+ assert offset <= end && offset >= this.offset && end <= this.end;
+ assert offset + shiftBy >= 0;
+ ByteBuffer sliceBuf = chunk.slice();
+ int newPos = (int) (offset - this.offset);
+ int newLimit = newPos + (int) (end - offset);
+ try {
+ sliceBuf.position(newPos);
+ sliceBuf.limit(newLimit);
+ } catch (Throwable t) {
+ LOG.error("Failed to slice buffer chunk with range" + " [" + this.offset + ", " + this.end
+ + "), position: " + chunk.position() + " limit: " + chunk.limit() + ", "
+ + (chunk.isDirect() ? "direct" : "array") + "; to [" + offset + ", " + end + ") "
+ + t.getClass());
+ throw new RuntimeException(t);
+ }
+ return new BufferChunk(sliceBuf, offset + shiftBy);
+ }
+
+ @Override
+ public ByteBuffer getData() {
+ return chunk;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java b/orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
new file mode 100644
index 0000000..745ed9a
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
@@ -0,0 +1,1097 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.orc.BinaryColumnStatistics;
+import org.apache.orc.BooleanColumnStatistics;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.DateColumnStatistics;
+import org.apache.orc.DecimalColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.OrcProto;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.TimestampColumnStatistics;
+import org.apache.orc.TypeDescription;
+
+public class ColumnStatisticsImpl implements ColumnStatistics {
+
+ private static final class BooleanStatisticsImpl extends ColumnStatisticsImpl
+ implements BooleanColumnStatistics {
+ private long trueCount = 0;
+
+ BooleanStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ super(stats);
+ OrcProto.BucketStatistics bkt = stats.getBucketStatistics();
+ trueCount = bkt.getCount(0);
+ }
+
+ BooleanStatisticsImpl() {
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ trueCount = 0;
+ }
+
+ @Override
+ public void updateBoolean(boolean value, int repetitions) {
+ if (value) {
+ trueCount += repetitions;
+ }
+ }
+
+ @Override
+ public void merge(ColumnStatisticsImpl other) {
+ if (other instanceof BooleanStatisticsImpl) {
+ BooleanStatisticsImpl bkt = (BooleanStatisticsImpl) other;
+ trueCount += bkt.trueCount;
+ } else {
+ if (isStatsExists() && trueCount != 0) {
+ throw new IllegalArgumentException("Incompatible merging of boolean column statistics");
+ }
+ }
+ super.merge(other);
+ }
+
+ @Override
+ public OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder builder = super.serialize();
+ OrcProto.BucketStatistics.Builder bucket =
+ OrcProto.BucketStatistics.newBuilder();
+ bucket.addCount(trueCount);
+ builder.setBucketStatistics(bucket);
+ return builder;
+ }
+
+ @Override
+ public long getFalseCount() {
+ return getNumberOfValues() - trueCount;
+ }
+
+ @Override
+ public long getTrueCount() {
+ return trueCount;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " true: " + trueCount;
+ }
+ }
+
+ private static final class IntegerStatisticsImpl extends ColumnStatisticsImpl
+ implements IntegerColumnStatistics {
+
+ private long minimum = Long.MAX_VALUE;
+ private long maximum = Long.MIN_VALUE;
+ private long sum = 0;
+ private boolean hasMinimum = false;
+ private boolean overflow = false;
+
+ IntegerStatisticsImpl() {
+ }
+
+ IntegerStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ super(stats);
+ OrcProto.IntegerStatistics intStat = stats.getIntStatistics();
+ if (intStat.hasMinimum()) {
+ hasMinimum = true;
+ minimum = intStat.getMinimum();
+ }
+ if (intStat.hasMaximum()) {
+ maximum = intStat.getMaximum();
+ }
+ if (intStat.hasSum()) {
+ sum = intStat.getSum();
+ } else {
+ overflow = true;
+ }
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ hasMinimum = false;
+ minimum = Long.MAX_VALUE;
+ maximum = Long.MIN_VALUE;
+ sum = 0;
+ overflow = false;
+ }
+
+ @Override
+ public void updateInteger(long value, int repetitions) {
+ if (!hasMinimum) {
+ hasMinimum = true;
+ minimum = value;
+ maximum = value;
+ } else if (value < minimum) {
+ minimum = value;
+ } else if (value > maximum) {
+ maximum = value;
+ }
+ if (!overflow) {
+ boolean wasPositive = sum >= 0;
+ sum += value * repetitions;
+ if ((value >= 0) == wasPositive) {
+ overflow = (sum >= 0) != wasPositive;
+ }
+ }
+ }
+
+ @Override
+ public void merge(ColumnStatisticsImpl other) {
+ if (other instanceof IntegerStatisticsImpl) {
+ IntegerStatisticsImpl otherInt = (IntegerStatisticsImpl) other;
+ if (!hasMinimum) {
+ hasMinimum = otherInt.hasMinimum;
+ minimum = otherInt.minimum;
+ maximum = otherInt.maximum;
+ } else if (otherInt.hasMinimum) {
+ if (otherInt.minimum < minimum) {
+ minimum = otherInt.minimum;
+ }
+ if (otherInt.maximum > maximum) {
+ maximum = otherInt.maximum;
+ }
+ }
+
+ overflow |= otherInt.overflow;
+ if (!overflow) {
+ boolean wasPositive = sum >= 0;
+ sum += otherInt.sum;
+ if ((otherInt.sum >= 0) == wasPositive) {
+ overflow = (sum >= 0) != wasPositive;
+ }
+ }
+ } else {
+ if (isStatsExists() && hasMinimum) {
+ throw new IllegalArgumentException("Incompatible merging of integer column statistics");
+ }
+ }
+ super.merge(other);
+ }
+
+ @Override
+ public OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder builder = super.serialize();
+ OrcProto.IntegerStatistics.Builder intb =
+ OrcProto.IntegerStatistics.newBuilder();
+ if (hasMinimum) {
+ intb.setMinimum(minimum);
+ intb.setMaximum(maximum);
+ }
+ if (!overflow) {
+ intb.setSum(sum);
+ }
+ builder.setIntStatistics(intb);
+ return builder;
+ }
+
+ @Override
+ public long getMinimum() {
+ return minimum;
+ }
+
+ @Override
+ public long getMaximum() {
+ return maximum;
+ }
+
+ @Override
+ public boolean isSumDefined() {
+ return !overflow;
+ }
+
+ @Override
+ public long getSum() {
+ return sum;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (hasMinimum) {
+ buf.append(" min: ");
+ buf.append(minimum);
+ buf.append(" max: ");
+ buf.append(maximum);
+ }
+ if (!overflow) {
+ buf.append(" sum: ");
+ buf.append(sum);
+ }
+ return buf.toString();
+ }
+ }
+
+ private static final class DoubleStatisticsImpl extends ColumnStatisticsImpl
+ implements DoubleColumnStatistics {
+ private boolean hasMinimum = false;
+ private double minimum = Double.MAX_VALUE;
+ private double maximum = Double.MIN_VALUE;
+ private double sum = 0;
+
+ DoubleStatisticsImpl() {
+ }
+
+ DoubleStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ super(stats);
+ OrcProto.DoubleStatistics dbl = stats.getDoubleStatistics();
+ if (dbl.hasMinimum()) {
+ hasMinimum = true;
+ minimum = dbl.getMinimum();
+ }
+ if (dbl.hasMaximum()) {
+ maximum = dbl.getMaximum();
+ }
+ if (dbl.hasSum()) {
+ sum = dbl.getSum();
+ }
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ hasMinimum = false;
+ minimum = Double.MAX_VALUE;
+ maximum = Double.MIN_VALUE;
+ sum = 0;
+ }
+
+ @Override
+ public void updateDouble(double value) {
+ if (!hasMinimum) {
+ hasMinimum = true;
+ minimum = value;
+ maximum = value;
+ } else if (value < minimum) {
+ minimum = value;
+ } else if (value > maximum) {
+ maximum = value;
+ }
+ sum += value;
+ }
+
+ @Override
+ public void merge(ColumnStatisticsImpl other) {
+ if (other instanceof DoubleStatisticsImpl) {
+ DoubleStatisticsImpl dbl = (DoubleStatisticsImpl) other;
+ if (!hasMinimum) {
+ hasMinimum = dbl.hasMinimum;
+ minimum = dbl.minimum;
+ maximum = dbl.maximum;
+ } else if (dbl.hasMinimum) {
+ if (dbl.minimum < minimum) {
+ minimum = dbl.minimum;
+ }
+ if (dbl.maximum > maximum) {
+ maximum = dbl.maximum;
+ }
+ }
+ sum += dbl.sum;
+ } else {
+ if (isStatsExists() && hasMinimum) {
+ throw new IllegalArgumentException("Incompatible merging of double column statistics");
+ }
+ }
+ super.merge(other);
+ }
+
+ @Override
+ public OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder builder = super.serialize();
+ OrcProto.DoubleStatistics.Builder dbl =
+ OrcProto.DoubleStatistics.newBuilder();
+ if (hasMinimum) {
+ dbl.setMinimum(minimum);
+ dbl.setMaximum(maximum);
+ }
+ dbl.setSum(sum);
+ builder.setDoubleStatistics(dbl);
+ return builder;
+ }
+
+ @Override
+ public double getMinimum() {
+ return minimum;
+ }
+
+ @Override
+ public double getMaximum() {
+ return maximum;
+ }
+
+ @Override
+ public double getSum() {
+ return sum;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (hasMinimum) {
+ buf.append(" min: ");
+ buf.append(minimum);
+ buf.append(" max: ");
+ buf.append(maximum);
+ }
+ buf.append(" sum: ");
+ buf.append(sum);
+ return buf.toString();
+ }
+ }
+
+ protected static final class StringStatisticsImpl extends ColumnStatisticsImpl
+ implements StringColumnStatistics {
+ private Text minimum = null;
+ private Text maximum = null;
+ private long sum = 0;
+
+ StringStatisticsImpl() {
+ }
+
+ StringStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ super(stats);
+ OrcProto.StringStatistics str = stats.getStringStatistics();
+ if (str.hasMaximum()) {
+ maximum = new Text(str.getMaximum());
+ }
+ if (str.hasMinimum()) {
+ minimum = new Text(str.getMinimum());
+ }
+ if(str.hasSum()) {
+ sum = str.getSum();
+ }
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ minimum = null;
+ maximum = null;
+ sum = 0;
+ }
+
+ @Override
+ public void updateString(Text value) {
+ if (minimum == null) {
+ maximum = minimum = new Text(value);
+ } else if (minimum.compareTo(value) > 0) {
+ minimum = new Text(value);
+ } else if (maximum.compareTo(value) < 0) {
+ maximum = new Text(value);
+ }
+ sum += value.getLength();
+ }
+
+ @Override
+ public void updateString(byte[] bytes, int offset, int length,
+ int repetitions) {
+ if (minimum == null) {
+ maximum = minimum = new Text();
+ maximum.set(bytes, offset, length);
+ } else if (WritableComparator.compareBytes(minimum.getBytes(), 0,
+ minimum.getLength(), bytes, offset, length) > 0) {
+ minimum = new Text();
+ minimum.set(bytes, offset, length);
+ } else if (WritableComparator.compareBytes(maximum.getBytes(), 0,
+ maximum.getLength(), bytes, offset, length) < 0) {
+ maximum = new Text();
+ maximum.set(bytes, offset, length);
+ }
+ sum += length * repetitions;
+ }
+
+ @Override
+ public void merge(ColumnStatisticsImpl other) {
+ if (other instanceof StringStatisticsImpl) {
+ StringStatisticsImpl str = (StringStatisticsImpl) other;
+ if (minimum == null) {
+ if (str.minimum != null) {
+ maximum = new Text(str.getMaximum());
+ minimum = new Text(str.getMinimum());
+ } else {
+ /* both are empty */
+ maximum = minimum = null;
+ }
+ } else if (str.minimum != null) {
+ if (minimum.compareTo(str.minimum) > 0) {
+ minimum = new Text(str.getMinimum());
+ }
+ if (maximum.compareTo(str.maximum) < 0) {
+ maximum = new Text(str.getMaximum());
+ }
+ }
+ sum += str.sum;
+ } else {
+ if (isStatsExists() && minimum != null) {
+ throw new IllegalArgumentException("Incompatible merging of string column statistics");
+ }
+ }
+ super.merge(other);
+ }
+
+ @Override
+ public OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder result = super.serialize();
+ OrcProto.StringStatistics.Builder str =
+ OrcProto.StringStatistics.newBuilder();
+ if (getNumberOfValues() != 0) {
+ str.setMinimum(getMinimum());
+ str.setMaximum(getMaximum());
+ str.setSum(sum);
+ }
+ result.setStringStatistics(str);
+ return result;
+ }
+
+ @Override
+ public String getMinimum() {
+ return minimum == null ? null : minimum.toString();
+ }
+
+ @Override
+ public String getMaximum() {
+ return maximum == null ? null : maximum.toString();
+ }
+
+ @Override
+ public long getSum() {
+ return sum;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (getNumberOfValues() != 0) {
+ buf.append(" min: ");
+ buf.append(getMinimum());
+ buf.append(" max: ");
+ buf.append(getMaximum());
+ buf.append(" sum: ");
+ buf.append(sum);
+ }
+ return buf.toString();
+ }
+ }
+
+ protected static final class BinaryStatisticsImpl extends ColumnStatisticsImpl implements
+ BinaryColumnStatistics {
+
+ private long sum = 0;
+
+ BinaryStatisticsImpl() {
+ }
+
+ BinaryStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ super(stats);
+ OrcProto.BinaryStatistics binStats = stats.getBinaryStatistics();
+ if (binStats.hasSum()) {
+ sum = binStats.getSum();
+ }
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ sum = 0;
+ }
+
+ @Override
+ public void updateBinary(BytesWritable value) {
+ sum += value.getLength();
+ }
+
+ @Override
+ public void updateBinary(byte[] bytes, int offset, int length,
+ int repetitions) {
+ sum += length * repetitions;
+ }
+
+ @Override
+ public void merge(ColumnStatisticsImpl other) {
+ if (other instanceof BinaryColumnStatistics) {
+ BinaryStatisticsImpl bin = (BinaryStatisticsImpl) other;
+ sum += bin.sum;
+ } else {
+ if (isStatsExists() && sum != 0) {
+ throw new IllegalArgumentException("Incompatible merging of binary column statistics");
+ }
+ }
+ super.merge(other);
+ }
+
+ @Override
+ public long getSum() {
+ return sum;
+ }
+
+ @Override
+ public OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder result = super.serialize();
+ OrcProto.BinaryStatistics.Builder bin = OrcProto.BinaryStatistics.newBuilder();
+ bin.setSum(sum);
+ result.setBinaryStatistics(bin);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (getNumberOfValues() != 0) {
+ buf.append(" sum: ");
+ buf.append(sum);
+ }
+ return buf.toString();
+ }
+ }
+
+ private static final class DecimalStatisticsImpl extends ColumnStatisticsImpl
+ implements DecimalColumnStatistics {
+ private HiveDecimal minimum = null;
+ private HiveDecimal maximum = null;
+ private HiveDecimal sum = HiveDecimal.ZERO;
+
+ DecimalStatisticsImpl() {
+ }
+
+ DecimalStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ super(stats);
+ OrcProto.DecimalStatistics dec = stats.getDecimalStatistics();
+ if (dec.hasMaximum()) {
+ maximum = HiveDecimal.create(dec.getMaximum());
+ }
+ if (dec.hasMinimum()) {
+ minimum = HiveDecimal.create(dec.getMinimum());
+ }
+ if (dec.hasSum()) {
+ sum = HiveDecimal.create(dec.getSum());
+ } else {
+ sum = null;
+ }
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ minimum = null;
+ maximum = null;
+ sum = HiveDecimal.ZERO;
+ }
+
+ @Override
+ public void updateDecimal(HiveDecimal value) {
+ if (minimum == null) {
+ minimum = value;
+ maximum = value;
+ } else if (minimum.compareTo(value) > 0) {
+ minimum = value;
+ } else if (maximum.compareTo(value) < 0) {
+ maximum = value;
+ }
+ if (sum != null) {
+ sum = sum.add(value);
+ }
+ }
+
+ @Override
+ public void merge(ColumnStatisticsImpl other) {
+ if (other instanceof DecimalStatisticsImpl) {
+ DecimalStatisticsImpl dec = (DecimalStatisticsImpl) other;
+ if (minimum == null) {
+ minimum = dec.minimum;
+ maximum = dec.maximum;
+ sum = dec.sum;
+ } else if (dec.minimum != null) {
+ if (minimum.compareTo(dec.minimum) > 0) {
+ minimum = dec.minimum;
+ }
+ if (maximum.compareTo(dec.maximum) < 0) {
+ maximum = dec.maximum;
+ }
+ if (sum == null || dec.sum == null) {
+ sum = null;
+ } else {
+ sum = sum.add(dec.sum);
+ }
+ }
+ } else {
+ if (isStatsExists() && minimum != null) {
+ throw new IllegalArgumentException("Incompatible merging of decimal column statistics");
+ }
+ }
+ super.merge(other);
+ }
+
+ @Override
+ public OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder result = super.serialize();
+ OrcProto.DecimalStatistics.Builder dec =
+ OrcProto.DecimalStatistics.newBuilder();
+ if (getNumberOfValues() != 0 && minimum != null) {
+ dec.setMinimum(minimum.toString());
+ dec.setMaximum(maximum.toString());
+ }
+ if (sum != null) {
+ dec.setSum(sum.toString());
+ }
+ result.setDecimalStatistics(dec);
+ return result;
+ }
+
+ @Override
+ public HiveDecimal getMinimum() {
+ return minimum;
+ }
+
+ @Override
+ public HiveDecimal getMaximum() {
+ return maximum;
+ }
+
+ @Override
+ public HiveDecimal getSum() {
+ return sum;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (getNumberOfValues() != 0) {
+ buf.append(" min: ");
+ buf.append(minimum);
+ buf.append(" max: ");
+ buf.append(maximum);
+ if (sum != null) {
+ buf.append(" sum: ");
+ buf.append(sum);
+ }
+ }
+ return buf.toString();
+ }
+ }
+
+ private static final class DateStatisticsImpl extends ColumnStatisticsImpl
+ implements DateColumnStatistics {
+ private Integer minimum = null;
+ private Integer maximum = null;
+
+ DateStatisticsImpl() {
+ }
+
+ DateStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ super(stats);
+ OrcProto.DateStatistics dateStats = stats.getDateStatistics();
+ // min,max values serialized/deserialized as int (days since epoch)
+ if (dateStats.hasMaximum()) {
+ maximum = dateStats.getMaximum();
+ }
+ if (dateStats.hasMinimum()) {
+ minimum = dateStats.getMinimum();
+ }
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ minimum = null;
+ maximum = null;
+ }
+
+ @Override
+ public void updateDate(DateWritable value) {
+ if (minimum == null) {
+ minimum = value.getDays();
+ maximum = value.getDays();
+ } else if (minimum > value.getDays()) {
+ minimum = value.getDays();
+ } else if (maximum < value.getDays()) {
+ maximum = value.getDays();
+ }
+ }
+
+ @Override
+ public void updateDate(int value) {
+ if (minimum == null) {
+ minimum = value;
+ maximum = value;
+ } else if (minimum > value) {
+ minimum = value;
+ } else if (maximum < value) {
+ maximum = value;
+ }
+ }
+
+ @Override
+ public void merge(ColumnStatisticsImpl other) {
+ if (other instanceof DateStatisticsImpl) {
+ DateStatisticsImpl dateStats = (DateStatisticsImpl) other;
+ if (minimum == null) {
+ minimum = dateStats.minimum;
+ maximum = dateStats.maximum;
+ } else if (dateStats.minimum != null) {
+ if (minimum > dateStats.minimum) {
+ minimum = dateStats.minimum;
+ }
+ if (maximum < dateStats.maximum) {
+ maximum = dateStats.maximum;
+ }
+ }
+ } else {
+ if (isStatsExists() && minimum != null) {
+ throw new IllegalArgumentException("Incompatible merging of date column statistics");
+ }
+ }
+ super.merge(other);
+ }
+
+ @Override
+ public OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder result = super.serialize();
+ OrcProto.DateStatistics.Builder dateStats =
+ OrcProto.DateStatistics.newBuilder();
+ if (getNumberOfValues() != 0 && minimum != null) {
+ dateStats.setMinimum(minimum);
+ dateStats.setMaximum(maximum);
+ }
+ result.setDateStatistics(dateStats);
+ return result;
+ }
+
+ private transient final DateWritable minDate = new DateWritable();
+ private transient final DateWritable maxDate = new DateWritable();
+
+ @Override
+ public Date getMinimum() {
+ if (minimum == null) {
+ return null;
+ }
+ minDate.set(minimum);
+ return minDate.get();
+ }
+
+ @Override
+ public Date getMaximum() {
+ if (maximum == null) {
+ return null;
+ }
+ maxDate.set(maximum);
+ return maxDate.get();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (getNumberOfValues() != 0) {
+ buf.append(" min: ");
+ buf.append(getMinimum());
+ buf.append(" max: ");
+ buf.append(getMaximum());
+ }
+ return buf.toString();
+ }
+ }
+
+ private static final class TimestampStatisticsImpl extends ColumnStatisticsImpl
+ implements TimestampColumnStatistics {
+ private Long minimum = null;
+ private Long maximum = null;
+
+ TimestampStatisticsImpl() {
+ }
+
+ TimestampStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ super(stats);
+ OrcProto.TimestampStatistics timestampStats = stats.getTimestampStatistics();
+ // min,max values serialized/deserialized as int (milliseconds since epoch)
+ if (timestampStats.hasMaximum()) {
+ maximum = timestampStats.getMaximum();
+ }
+ if (timestampStats.hasMinimum()) {
+ minimum = timestampStats.getMinimum();
+ }
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ minimum = null;
+ maximum = null;
+ }
+
+ @Override
+ public void updateTimestamp(Timestamp value) {
+ if (minimum == null) {
+ minimum = value.getTime();
+ maximum = value.getTime();
+ } else if (minimum > value.getTime()) {
+ minimum = value.getTime();
+ } else if (maximum < value.getTime()) {
+ maximum = value.getTime();
+ }
+ }
+
+ @Override
+ public void updateTimestamp(long value) {
+ if (minimum == null) {
+ minimum = value;
+ maximum = value;
+ } else if (minimum > value) {
+ minimum = value;
+ } else if (maximum < value) {
+ maximum = value;
+ }
+ }
+
+ @Override
+ public void merge(ColumnStatisticsImpl other) {
+ if (other instanceof TimestampStatisticsImpl) {
+ TimestampStatisticsImpl timestampStats = (TimestampStatisticsImpl) other;
+ if (minimum == null) {
+ minimum = timestampStats.minimum;
+ maximum = timestampStats.maximum;
+ } else if (timestampStats.minimum != null) {
+ if (minimum > timestampStats.minimum) {
+ minimum = timestampStats.minimum;
+ }
+ if (maximum < timestampStats.maximum) {
+ maximum = timestampStats.maximum;
+ }
+ }
+ } else {
+ if (isStatsExists() && minimum != null) {
+ throw new IllegalArgumentException("Incompatible merging of timestamp column statistics");
+ }
+ }
+ super.merge(other);
+ }
+
+ @Override
+ public OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder result = super.serialize();
+ OrcProto.TimestampStatistics.Builder timestampStats = OrcProto.TimestampStatistics
+ .newBuilder();
+ if (getNumberOfValues() != 0 && minimum != null) {
+ timestampStats.setMinimum(minimum);
+ timestampStats.setMaximum(maximum);
+ }
+ result.setTimestampStatistics(timestampStats);
+ return result;
+ }
+
+ @Override
+ public Timestamp getMinimum() {
+ return minimum == null ? null : new Timestamp(minimum);
+ }
+
+ @Override
+ public Timestamp getMaximum() {
+ return maximum == null ? null : new Timestamp(maximum);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (getNumberOfValues() != 0) {
+ buf.append(" min: ");
+ buf.append(getMinimum());
+ buf.append(" max: ");
+ buf.append(getMaximum());
+ }
+ return buf.toString();
+ }
+ }
+
+ private long count = 0;
+ private boolean hasNull = false;
+
+ ColumnStatisticsImpl(OrcProto.ColumnStatistics stats) {
+ if (stats.hasNumberOfValues()) {
+ count = stats.getNumberOfValues();
+ }
+
+ if (stats.hasHasNull()) {
+ hasNull = stats.getHasNull();
+ } else {
+ hasNull = true;
+ }
+ }
+
+ ColumnStatisticsImpl() {
+ }
+
+ public void increment() {
+ count += 1;
+ }
+
+ public void increment(int count) {
+ this.count += count;
+ }
+
+ public void setNull() {
+ hasNull = true;
+ }
+
+ public void updateBoolean(boolean value, int repetitions) {
+ throw new UnsupportedOperationException("Can't update boolean");
+ }
+
+ public void updateInteger(long value, int repetitions) {
+ throw new UnsupportedOperationException("Can't update integer");
+ }
+
+ public void updateDouble(double value) {
+ throw new UnsupportedOperationException("Can't update double");
+ }
+
+ public void updateString(Text value) {
+ throw new UnsupportedOperationException("Can't update string");
+ }
+
+ public void updateString(byte[] bytes, int offset, int length,
+ int repetitions) {
+ throw new UnsupportedOperationException("Can't update string");
+ }
+
+ public void updateBinary(BytesWritable value) {
+ throw new UnsupportedOperationException("Can't update binary");
+ }
+
+ public void updateBinary(byte[] bytes, int offset, int length,
+ int repetitions) {
+ throw new UnsupportedOperationException("Can't update string");
+ }
+
+ public void updateDecimal(HiveDecimal value) {
+ throw new UnsupportedOperationException("Can't update decimal");
+ }
+
+ public void updateDate(DateWritable value) {
+ throw new UnsupportedOperationException("Can't update date");
+ }
+
+ public void updateDate(int value) {
+ throw new UnsupportedOperationException("Can't update date");
+ }
+
+ public void updateTimestamp(Timestamp value) {
+ throw new UnsupportedOperationException("Can't update timestamp");
+ }
+
+ public void updateTimestamp(long value) {
+ throw new UnsupportedOperationException("Can't update timestamp");
+ }
+
+ public boolean isStatsExists() {
+ return (count > 0 || hasNull == true);
+ }
+
+ public void merge(ColumnStatisticsImpl stats) {
+ count += stats.count;
+ hasNull |= stats.hasNull;
+ }
+
+ public void reset() {
+ count = 0;
+ hasNull = false;
+ }
+
+ @Override
+ public long getNumberOfValues() {
+ return count;
+ }
+
+ @Override
+ public boolean hasNull() {
+ return hasNull;
+ }
+
+ @Override
+ public String toString() {
+ return "count: " + count + " hasNull: " + hasNull;
+ }
+
+ public OrcProto.ColumnStatistics.Builder serialize() {
+ OrcProto.ColumnStatistics.Builder builder =
+ OrcProto.ColumnStatistics.newBuilder();
+ builder.setNumberOfValues(count);
+ builder.setHasNull(hasNull);
+ return builder;
+ }
+
+ public static ColumnStatisticsImpl create(TypeDescription schema) {
+ switch (schema.getCategory()) {
+ case BOOLEAN:
+ return new BooleanStatisticsImpl();
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ return new IntegerStatisticsImpl();
+ case FLOAT:
+ case DOUBLE:
+ return new DoubleStatisticsImpl();
+ case STRING:
+ case CHAR:
+ case VARCHAR:
+ return new StringStatisticsImpl();
+ case DECIMAL:
+ return new DecimalStatisticsImpl();
+ case DATE:
+ return new DateStatisticsImpl();
+ case TIMESTAMP:
+ return new TimestampStatisticsImpl();
+ case BINARY:
+ return new BinaryStatisticsImpl();
+ default:
+ return new ColumnStatisticsImpl();
+ }
+ }
+
+ public static ColumnStatisticsImpl deserialize(OrcProto.ColumnStatistics stats) {
+ if (stats.hasBucketStatistics()) {
+ return new BooleanStatisticsImpl(stats);
+ } else if (stats.hasIntStatistics()) {
+ return new IntegerStatisticsImpl(stats);
+ } else if (stats.hasDoubleStatistics()) {
+ return new DoubleStatisticsImpl(stats);
+ } else if (stats.hasStringStatistics()) {
+ return new StringStatisticsImpl(stats);
+ } else if (stats.hasDecimalStatistics()) {
+ return new DecimalStatisticsImpl(stats);
+ } else if (stats.hasDateStatistics()) {
+ return new DateStatisticsImpl(stats);
+ } else if (stats.hasTimestampStatistics()) {
+ return new TimestampStatisticsImpl(stats);
+ } else if(stats.hasBinaryStatistics()) {
+ return new BinaryStatisticsImpl(stats);
+ } else {
+ return new ColumnStatisticsImpl(stats);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/DirectDecompressionCodec.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/DirectDecompressionCodec.java b/orc/src/java/org/apache/orc/impl/DirectDecompressionCodec.java
new file mode 100644
index 0000000..7e0110d
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/DirectDecompressionCodec.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import org.apache.orc.CompressionCodec;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface DirectDecompressionCodec extends CompressionCodec {
+ public boolean isAvailable();
+ public void directDecompress(ByteBuffer in, ByteBuffer out) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/DynamicByteArray.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/DynamicByteArray.java b/orc/src/java/org/apache/orc/impl/DynamicByteArray.java
new file mode 100644
index 0000000..986c2ac
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/DynamicByteArray.java
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.io.Text;
+
+/**
+ * A class that is a growable array of bytes. Growth is managed in terms of
+ * chunks that are allocated when needed.
+ */
+public final class DynamicByteArray {
+ static final int DEFAULT_CHUNKSIZE = 32 * 1024;
+ static final int DEFAULT_NUM_CHUNKS = 128;
+
+ private final int chunkSize; // our allocation sizes
+ private byte[][] data; // the real data
+ private int length; // max set element index +1
+ private int initializedChunks = 0; // the number of chunks created
+
+ public DynamicByteArray() {
+ this(DEFAULT_NUM_CHUNKS, DEFAULT_CHUNKSIZE);
+ }
+
+ public DynamicByteArray(int numChunks, int chunkSize) {
+ if (chunkSize == 0) {
+ throw new IllegalArgumentException("bad chunksize");
+ }
+ this.chunkSize = chunkSize;
+ data = new byte[numChunks][];
+ }
+
+ /**
+ * Ensure that the given index is valid.
+ */
+ private void grow(int chunkIndex) {
+ if (chunkIndex >= initializedChunks) {
+ if (chunkIndex >= data.length) {
+ int newSize = Math.max(chunkIndex + 1, 2 * data.length);
+ byte[][] newChunk = new byte[newSize][];
+ System.arraycopy(data, 0, newChunk, 0, data.length);
+ data = newChunk;
+ }
+ for(int i=initializedChunks; i <= chunkIndex; ++i) {
+ data[i] = new byte[chunkSize];
+ }
+ initializedChunks = chunkIndex + 1;
+ }
+ }
+
+ public byte get(int index) {
+ if (index >= length) {
+ throw new IndexOutOfBoundsException("Index " + index +
+ " is outside of 0.." +
+ (length - 1));
+ }
+ int i = index / chunkSize;
+ int j = index % chunkSize;
+ return data[i][j];
+ }
+
+ public void set(int index, byte value) {
+ int i = index / chunkSize;
+ int j = index % chunkSize;
+ grow(i);
+ if (index >= length) {
+ length = index + 1;
+ }
+ data[i][j] = value;
+ }
+
+ public int add(byte value) {
+ int i = length / chunkSize;
+ int j = length % chunkSize;
+ grow(i);
+ data[i][j] = value;
+ int result = length;
+ length += 1;
+ return result;
+ }
+
+ /**
+ * Copy a slice of a byte array into our buffer.
+ * @param value the array to copy from
+ * @param valueOffset the first location to copy from value
+ * @param valueLength the number of bytes to copy from value
+ * @return the offset of the start of the value
+ */
+ public int add(byte[] value, int valueOffset, int valueLength) {
+ int i = length / chunkSize;
+ int j = length % chunkSize;
+ grow((length + valueLength) / chunkSize);
+ int remaining = valueLength;
+ while (remaining > 0) {
+ int size = Math.min(remaining, chunkSize - j);
+ System.arraycopy(value, valueOffset, data[i], j, size);
+ remaining -= size;
+ valueOffset += size;
+ i += 1;
+ j = 0;
+ }
+ int result = length;
+ length += valueLength;
+ return result;
+ }
+
+ /**
+ * Read the entire stream into this array.
+ * @param in the stream to read from
+ * @throws IOException
+ */
+ public void readAll(InputStream in) throws IOException {
+ int currentChunk = length / chunkSize;
+ int currentOffset = length % chunkSize;
+ grow(currentChunk);
+ int currentLength = in.read(data[currentChunk], currentOffset,
+ chunkSize - currentOffset);
+ while (currentLength > 0) {
+ length += currentLength;
+ currentOffset = length % chunkSize;
+ if (currentOffset == 0) {
+ currentChunk = length / chunkSize;
+ grow(currentChunk);
+ }
+ currentLength = in.read(data[currentChunk], currentOffset,
+ chunkSize - currentOffset);
+ }
+ }
+
+ /**
+ * Byte compare a set of bytes against the bytes in this dynamic array.
+ * @param other source of the other bytes
+ * @param otherOffset start offset in the other array
+ * @param otherLength number of bytes in the other array
+ * @param ourOffset the offset in our array
+ * @param ourLength the number of bytes in our array
+ * @return negative for less, 0 for equal, positive for greater
+ */
+ public int compare(byte[] other, int otherOffset, int otherLength,
+ int ourOffset, int ourLength) {
+ int currentChunk = ourOffset / chunkSize;
+ int currentOffset = ourOffset % chunkSize;
+ int maxLength = Math.min(otherLength, ourLength);
+ while (maxLength > 0 &&
+ other[otherOffset] == data[currentChunk][currentOffset]) {
+ otherOffset += 1;
+ currentOffset += 1;
+ if (currentOffset == chunkSize) {
+ currentChunk += 1;
+ currentOffset = 0;
+ }
+ maxLength -= 1;
+ }
+ if (maxLength == 0) {
+ return otherLength - ourLength;
+ }
+ int otherByte = 0xff & other[otherOffset];
+ int ourByte = 0xff & data[currentChunk][currentOffset];
+ return otherByte > ourByte ? 1 : -1;
+ }
+
+ /**
+ * Get the size of the array.
+ * @return the number of bytes in the array
+ */
+ public int size() {
+ return length;
+ }
+
+ /**
+ * Clear the array to its original pristine state.
+ */
+ public void clear() {
+ length = 0;
+ for(int i=0; i < data.length; ++i) {
+ data[i] = null;
+ }
+ initializedChunks = 0;
+ }
+
+ /**
+ * Set a text value from the bytes in this dynamic array.
+ * @param result the value to set
+ * @param offset the start of the bytes to copy
+ * @param length the number of bytes to copy
+ */
+ public void setText(Text result, int offset, int length) {
+ result.clear();
+ int currentChunk = offset / chunkSize;
+ int currentOffset = offset % chunkSize;
+ int currentLength = Math.min(length, chunkSize - currentOffset);
+ while (length > 0) {
+ result.append(data[currentChunk], currentOffset, currentLength);
+ length -= currentLength;
+ currentChunk += 1;
+ currentOffset = 0;
+ currentLength = Math.min(length, chunkSize - currentOffset);
+ }
+ }
+
+ /**
+ * Write out a range of this dynamic array to an output stream.
+ * @param out the stream to write to
+ * @param offset the first offset to write
+ * @param length the number of bytes to write
+ * @throws IOException
+ */
+ public void write(OutputStream out, int offset,
+ int length) throws IOException {
+ int currentChunk = offset / chunkSize;
+ int currentOffset = offset % chunkSize;
+ while (length > 0) {
+ int currentLength = Math.min(length, chunkSize - currentOffset);
+ out.write(data[currentChunk], currentOffset, currentLength);
+ length -= currentLength;
+ currentChunk += 1;
+ currentOffset = 0;
+ }
+ }
+
+ @Override
+ public String toString() {
+ int i;
+ StringBuilder sb = new StringBuilder(length * 3);
+
+ sb.append('{');
+ int l = length - 1;
+ for (i=0; i<l; i++) {
+ sb.append(Integer.toHexString(get(i)));
+ sb.append(',');
+ }
+ sb.append(get(i));
+ sb.append('}');
+
+ return sb.toString();
+ }
+
+ public void setByteBuffer(ByteBuffer result, int offset, int length) {
+ result.clear();
+ int currentChunk = offset / chunkSize;
+ int currentOffset = offset % chunkSize;
+ int currentLength = Math.min(length, chunkSize - currentOffset);
+ while (length > 0) {
+ result.put(data[currentChunk], currentOffset, currentLength);
+ length -= currentLength;
+ currentChunk += 1;
+ currentOffset = 0;
+ currentLength = Math.min(length, chunkSize - currentOffset);
+ }
+ }
+
+ /**
+ * Gets all the bytes of the array.
+ *
+ * @return Bytes of the array
+ */
+ public byte[] get() {
+ byte[] result = null;
+ if (length > 0) {
+ int currentChunk = 0;
+ int currentOffset = 0;
+ int currentLength = Math.min(length, chunkSize);
+ int destOffset = 0;
+ result = new byte[length];
+ int totalLength = length;
+ while (totalLength > 0) {
+ System.arraycopy(data[currentChunk], currentOffset, result, destOffset, currentLength);
+ destOffset += currentLength;
+ totalLength -= currentLength;
+ currentChunk += 1;
+ currentOffset = 0;
+ currentLength = Math.min(totalLength, chunkSize - currentOffset);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Get the size of the buffers.
+ */
+ public long getSizeInBytes() {
+ return initializedChunks * chunkSize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/DynamicIntArray.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/DynamicIntArray.java b/orc/src/java/org/apache/orc/impl/DynamicIntArray.java
new file mode 100644
index 0000000..3b2884b
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/DynamicIntArray.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+/**
+ * Dynamic int array that uses primitive types and chunks to avoid copying
+ * large number of integers when it resizes.
+ *
+ * The motivation for this class is memory optimization, i.e. space efficient
+ * storage of potentially huge arrays without good a-priori size guesses.
+ *
+ * The API of this class is between a primitive array and a AbstractList. It's
+ * not a Collection implementation because it handles primitive types, but the
+ * API could be extended to support iterators and the like.
+ *
+ * NOTE: Like standard Collection implementations/arrays, this class is not
+ * synchronized.
+ */
+public final class DynamicIntArray {
+ static final int DEFAULT_CHUNKSIZE = 8 * 1024;
+ static final int INIT_CHUNKS = 128;
+
+ private final int chunkSize; // our allocation size
+ private int[][] data; // the real data
+ private int length; // max set element index +1
+ private int initializedChunks = 0; // the number of created chunks
+
+ public DynamicIntArray() {
+ this(DEFAULT_CHUNKSIZE);
+ }
+
+ public DynamicIntArray(int chunkSize) {
+ this.chunkSize = chunkSize;
+
+ data = new int[INIT_CHUNKS][];
+ }
+
+ /**
+ * Ensure that the given index is valid.
+ */
+ private void grow(int chunkIndex) {
+ if (chunkIndex >= initializedChunks) {
+ if (chunkIndex >= data.length) {
+ int newSize = Math.max(chunkIndex + 1, 2 * data.length);
+ int[][] newChunk = new int[newSize][];
+ System.arraycopy(data, 0, newChunk, 0, data.length);
+ data = newChunk;
+ }
+ for (int i=initializedChunks; i <= chunkIndex; ++i) {
+ data[i] = new int[chunkSize];
+ }
+ initializedChunks = chunkIndex + 1;
+ }
+ }
+
+ public int get(int index) {
+ if (index >= length) {
+ throw new IndexOutOfBoundsException("Index " + index +
+ " is outside of 0.." +
+ (length - 1));
+ }
+ int i = index / chunkSize;
+ int j = index % chunkSize;
+ return data[i][j];
+ }
+
+ public void set(int index, int value) {
+ int i = index / chunkSize;
+ int j = index % chunkSize;
+ grow(i);
+ if (index >= length) {
+ length = index + 1;
+ }
+ data[i][j] = value;
+ }
+
+ public void increment(int index, int value) {
+ int i = index / chunkSize;
+ int j = index % chunkSize;
+ grow(i);
+ if (index >= length) {
+ length = index + 1;
+ }
+ data[i][j] += value;
+ }
+
+ public void add(int value) {
+ int i = length / chunkSize;
+ int j = length % chunkSize;
+ grow(i);
+ data[i][j] = value;
+ length += 1;
+ }
+
+ public int size() {
+ return length;
+ }
+
+ public void clear() {
+ length = 0;
+ for(int i=0; i < data.length; ++i) {
+ data[i] = null;
+ }
+ initializedChunks = 0;
+ }
+
+ public String toString() {
+ int i;
+ StringBuilder sb = new StringBuilder(length * 4);
+
+ sb.append('{');
+ int l = length - 1;
+ for (i=0; i<l; i++) {
+ sb.append(get(i));
+ sb.append(',');
+ }
+ sb.append(get(i));
+ sb.append('}');
+
+ return sb.toString();
+ }
+
+ public int getSizeInBytes() {
+ return 4 * initializedChunks * chunkSize;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/HadoopShims.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/HadoopShims.java b/orc/src/java/org/apache/orc/impl/HadoopShims.java
new file mode 100644
index 0000000..2980d71
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/HadoopShims.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.hadoop.util.VersionInfo;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface HadoopShims {
+
+ enum DirectCompressionType {
+ NONE,
+ ZLIB_NOHEADER,
+ ZLIB,
+ SNAPPY,
+ }
+
+ interface DirectDecompressor {
+ void decompress(ByteBuffer var1, ByteBuffer var2) throws IOException;
+ }
+
+ /**
+ * Get a direct decompressor codec, if it is available
+ * @param codec
+ * @return
+ */
+ DirectDecompressor getDirectDecompressor(DirectCompressionType codec);
+
+
+ class Factory {
+ private static HadoopShims SHIMS = null;
+
+ public static synchronized HadoopShims get() {
+ if (SHIMS == null) {
+ String[] versionParts = VersionInfo.getVersion().split("[.]");
+ int major = Integer.parseInt(versionParts[0]);
+ int minor = Integer.parseInt(versionParts[1]);
+ if (major < 2 || (major == 2 && minor < 3)) {
+ SHIMS = new HadoopShims_2_2();
+ } else {
+ SHIMS = new HadoopShimsCurrent();
+ }
+ }
+ return SHIMS;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/HadoopShimsCurrent.java b/orc/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
new file mode 100644
index 0000000..3b9371d
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
+import org.apache.hadoop.io.compress.zlib.ZlibDecompressor;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Shims for recent versions of Hadoop
+ */
+public class HadoopShimsCurrent implements HadoopShims {
+
+ private static class DirectDecompressWrapper implements DirectDecompressor {
+ private final org.apache.hadoop.io.compress.DirectDecompressor root;
+
+ DirectDecompressWrapper(org.apache.hadoop.io.compress.DirectDecompressor root) {
+ this.root = root;
+ }
+
+ public void decompress(ByteBuffer input,
+ ByteBuffer output) throws IOException {
+ root.decompress(input, output);
+ }
+ }
+
+ public DirectDecompressor getDirectDecompressor(
+ DirectCompressionType codec) {
+ switch (codec) {
+ case ZLIB:
+ return new DirectDecompressWrapper
+ (new ZlibDecompressor.ZlibDirectDecompressor());
+ case ZLIB_NOHEADER:
+ return new DirectDecompressWrapper
+ (new ZlibDecompressor.ZlibDirectDecompressor
+ (ZlibDecompressor.CompressionHeader.NO_HEADER, 0));
+ case SNAPPY:
+ return new DirectDecompressWrapper
+ (new SnappyDecompressor.SnappyDirectDecompressor());
+ default:
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/HadoopShims_2_2.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/HadoopShims_2_2.java b/orc/src/java/org/apache/orc/impl/HadoopShims_2_2.java
new file mode 100644
index 0000000..ac46836
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/HadoopShims_2_2.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
+import org.apache.hadoop.io.compress.zlib.ZlibDecompressor;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Shims for versions of Hadoop up to and including 2.2.x
+ */
+public class HadoopShims_2_2 implements HadoopShims {
+
+ public DirectDecompressor getDirectDecompressor(
+ DirectCompressionType codec) {
+ return null;
+ }
+}
[05/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java
index 7412e6b..e36a744 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.UnionObject;
import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
+import org.apache.orc.OrcProto;
import java.util.ArrayList;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
deleted file mode 100644
index aabea0b..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
+++ /dev/null
@@ -1,629 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.io.IOConstants;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
-
-import com.google.common.collect.Lists;
-
-public class OrcUtils {
- private static final Logger LOG = LoggerFactory.getLogger(OrcUtils.class);
-
- /**
- * Returns selected columns as a boolean array with true value set for specified column names.
- * The result will contain number of elements equal to flattened number of columns.
- * For example:
- * selectedColumns - a,b,c
- * allColumns - a,b,c,d
- * If column c is a complex type, say list<string> and other types are primitives then result will
- * be [false, true, true, true, true, true, false]
- * Index 0 is the root element of the struct which is set to false by default, index 1,2
- * corresponds to columns a and b. Index 3,4 correspond to column c which is list<string> and
- * index 5 correspond to column d. After flattening list<string> gets 2 columns.
- *
- * @param selectedColumns - comma separated list of selected column names
- * @param schema - object schema
- * @return - boolean array with true value set for the specified column names
- */
- public static boolean[] includeColumns(String selectedColumns,
- TypeDescription schema) {
- int numFlattenedCols = schema.getMaximumId();
- boolean[] results = new boolean[numFlattenedCols + 1];
- if ("*".equals(selectedColumns)) {
- Arrays.fill(results, true);
- return results;
- }
- if (selectedColumns != null &&
- schema.getCategory() == TypeDescription.Category.STRUCT) {
- List<String> fieldNames = schema.getFieldNames();
- List<TypeDescription> fields = schema.getChildren();
- for (String column: selectedColumns.split((","))) {
- TypeDescription col = findColumn(column, fieldNames, fields);
- if (col != null) {
- for(int i=col.getId(); i <= col.getMaximumId(); ++i) {
- results[i] = true;
- }
- }
- }
- }
- return results;
- }
-
- private static TypeDescription findColumn(String columnName,
- List<String> fieldNames,
- List<TypeDescription> fields) {
- int i = 0;
- for(String fieldName: fieldNames) {
- if (fieldName.equalsIgnoreCase(columnName)) {
- return fields.get(i);
- } else {
- i += 1;
- }
- }
- return null;
- }
-
- /**
- * Convert a Hive type property string that contains separated type names into a list of
- * TypeDescription objects.
- * @return the list of TypeDescription objects.
- */
- public static ArrayList<TypeDescription> typeDescriptionsFromHiveTypeProperty(
- String hiveTypeProperty) {
-
- // CONSDIER: We need a type name parser for TypeDescription.
-
- ArrayList<TypeInfo> typeInfoList = TypeInfoUtils.getTypeInfosFromTypeString(hiveTypeProperty);
- ArrayList<TypeDescription> typeDescrList =new ArrayList<TypeDescription>(typeInfoList.size());
- for (TypeInfo typeInfo : typeInfoList) {
- typeDescrList.add(convertTypeInfo(typeInfo));
- }
- return typeDescrList;
- }
-
- public static TypeDescription convertTypeInfo(TypeInfo info) {
- switch (info.getCategory()) {
- case PRIMITIVE: {
- PrimitiveTypeInfo pinfo = (PrimitiveTypeInfo) info;
- switch (pinfo.getPrimitiveCategory()) {
- case BOOLEAN:
- return TypeDescription.createBoolean();
- case BYTE:
- return TypeDescription.createByte();
- case SHORT:
- return TypeDescription.createShort();
- case INT:
- return TypeDescription.createInt();
- case LONG:
- return TypeDescription.createLong();
- case FLOAT:
- return TypeDescription.createFloat();
- case DOUBLE:
- return TypeDescription.createDouble();
- case STRING:
- return TypeDescription.createString();
- case DATE:
- return TypeDescription.createDate();
- case TIMESTAMP:
- return TypeDescription.createTimestamp();
- case BINARY:
- return TypeDescription.createBinary();
- case DECIMAL: {
- DecimalTypeInfo dinfo = (DecimalTypeInfo) pinfo;
- return TypeDescription.createDecimal()
- .withScale(dinfo.getScale())
- .withPrecision(dinfo.getPrecision());
- }
- case VARCHAR: {
- BaseCharTypeInfo cinfo = (BaseCharTypeInfo) pinfo;
- return TypeDescription.createVarchar()
- .withMaxLength(cinfo.getLength());
- }
- case CHAR: {
- BaseCharTypeInfo cinfo = (BaseCharTypeInfo) pinfo;
- return TypeDescription.createChar()
- .withMaxLength(cinfo.getLength());
- }
- default:
- throw new IllegalArgumentException("ORC doesn't handle primitive" +
- " category " + pinfo.getPrimitiveCategory());
- }
- }
- case LIST: {
- ListTypeInfo linfo = (ListTypeInfo) info;
- return TypeDescription.createList
- (convertTypeInfo(linfo.getListElementTypeInfo()));
- }
- case MAP: {
- MapTypeInfo minfo = (MapTypeInfo) info;
- return TypeDescription.createMap
- (convertTypeInfo(minfo.getMapKeyTypeInfo()),
- convertTypeInfo(minfo.getMapValueTypeInfo()));
- }
- case UNION: {
- UnionTypeInfo minfo = (UnionTypeInfo) info;
- TypeDescription result = TypeDescription.createUnion();
- for (TypeInfo child: minfo.getAllUnionObjectTypeInfos()) {
- result.addUnionChild(convertTypeInfo(child));
- }
- return result;
- }
- case STRUCT: {
- StructTypeInfo sinfo = (StructTypeInfo) info;
- TypeDescription result = TypeDescription.createStruct();
- for(String fieldName: sinfo.getAllStructFieldNames()) {
- result.addField(fieldName,
- convertTypeInfo(sinfo.getStructFieldTypeInfo(fieldName)));
- }
- return result;
- }
- default:
- throw new IllegalArgumentException("ORC doesn't handle " +
- info.getCategory());
- }
- }
-
- public static List<OrcProto.Type> getOrcTypes(TypeDescription typeDescr) {
- List<OrcProto.Type> result = Lists.newArrayList();
- appendOrcTypes(result, typeDescr);
- return result;
- }
-
- private static void appendOrcTypes(List<OrcProto.Type> result, TypeDescription typeDescr) {
- OrcProto.Type.Builder type = OrcProto.Type.newBuilder();
- List<TypeDescription> children = typeDescr.getChildren();
- switch (typeDescr.getCategory()) {
- case BOOLEAN:
- type.setKind(OrcProto.Type.Kind.BOOLEAN);
- break;
- case BYTE:
- type.setKind(OrcProto.Type.Kind.BYTE);
- break;
- case SHORT:
- type.setKind(OrcProto.Type.Kind.SHORT);
- break;
- case INT:
- type.setKind(OrcProto.Type.Kind.INT);
- break;
- case LONG:
- type.setKind(OrcProto.Type.Kind.LONG);
- break;
- case FLOAT:
- type.setKind(OrcProto.Type.Kind.FLOAT);
- break;
- case DOUBLE:
- type.setKind(OrcProto.Type.Kind.DOUBLE);
- break;
- case STRING:
- type.setKind(OrcProto.Type.Kind.STRING);
- break;
- case CHAR:
- type.setKind(OrcProto.Type.Kind.CHAR);
- type.setMaximumLength(typeDescr.getMaxLength());
- break;
- case VARCHAR:
- type.setKind(Type.Kind.VARCHAR);
- type.setMaximumLength(typeDescr.getMaxLength());
- break;
- case BINARY:
- type.setKind(OrcProto.Type.Kind.BINARY);
- break;
- case TIMESTAMP:
- type.setKind(OrcProto.Type.Kind.TIMESTAMP);
- break;
- case DATE:
- type.setKind(OrcProto.Type.Kind.DATE);
- break;
- case DECIMAL:
- type.setKind(OrcProto.Type.Kind.DECIMAL);
- type.setPrecision(typeDescr.getPrecision());
- type.setScale(typeDescr.getScale());
- break;
- case LIST:
- type.setKind(OrcProto.Type.Kind.LIST);
- type.addSubtypes(children.get(0).getId());
- break;
- case MAP:
- type.setKind(OrcProto.Type.Kind.MAP);
- for(TypeDescription t: children) {
- type.addSubtypes(t.getId());
- }
- break;
- case STRUCT:
- type.setKind(OrcProto.Type.Kind.STRUCT);
- for(TypeDescription t: children) {
- type.addSubtypes(t.getId());
- }
- for(String field: typeDescr.getFieldNames()) {
- type.addFieldNames(field);
- }
- break;
- case UNION:
- type.setKind(OrcProto.Type.Kind.UNION);
- for(TypeDescription t: children) {
- type.addSubtypes(t.getId());
- }
- break;
- default:
- throw new IllegalArgumentException("Unknown category: " +
- typeDescr.getCategory());
- }
- result.add(type.build());
- if (children != null) {
- for(TypeDescription child: children) {
- appendOrcTypes(result, child);
- }
- }
- }
-
- /**
- * NOTE: This method ignores the subtype numbers in the TypeDescription rebuilds the subtype
- * numbers based on the length of the result list being appended.
- *
- * @param result
- * @param typeInfo
- */
- public static void appendOrcTypesRebuildSubtypes(List<OrcProto.Type> result,
- TypeDescription typeDescr) {
-
- int subtype = result.size();
- OrcProto.Type.Builder type = OrcProto.Type.newBuilder();
- boolean needsAdd = true;
- List<TypeDescription> children = typeDescr.getChildren();
- switch (typeDescr.getCategory()) {
- case BOOLEAN:
- type.setKind(OrcProto.Type.Kind.BOOLEAN);
- break;
- case BYTE:
- type.setKind(OrcProto.Type.Kind.BYTE);
- break;
- case SHORT:
- type.setKind(OrcProto.Type.Kind.SHORT);
- break;
- case INT:
- type.setKind(OrcProto.Type.Kind.INT);
- break;
- case LONG:
- type.setKind(OrcProto.Type.Kind.LONG);
- break;
- case FLOAT:
- type.setKind(OrcProto.Type.Kind.FLOAT);
- break;
- case DOUBLE:
- type.setKind(OrcProto.Type.Kind.DOUBLE);
- break;
- case STRING:
- type.setKind(OrcProto.Type.Kind.STRING);
- break;
- case CHAR:
- type.setKind(OrcProto.Type.Kind.CHAR);
- type.setMaximumLength(typeDescr.getMaxLength());
- break;
- case VARCHAR:
- type.setKind(Type.Kind.VARCHAR);
- type.setMaximumLength(typeDescr.getMaxLength());
- break;
- case BINARY:
- type.setKind(OrcProto.Type.Kind.BINARY);
- break;
- case TIMESTAMP:
- type.setKind(OrcProto.Type.Kind.TIMESTAMP);
- break;
- case DATE:
- type.setKind(OrcProto.Type.Kind.DATE);
- break;
- case DECIMAL:
- type.setKind(OrcProto.Type.Kind.DECIMAL);
- type.setPrecision(typeDescr.getPrecision());
- type.setScale(typeDescr.getScale());
- break;
- case LIST:
- type.setKind(OrcProto.Type.Kind.LIST);
- type.addSubtypes(++subtype);
- result.add(type.build());
- needsAdd = false;
- appendOrcTypesRebuildSubtypes(result, children.get(0));
- break;
- case MAP:
- {
- // Make room for MAP type.
- result.add(null);
-
- // Add MAP type pair in order to determine their subtype values.
- appendOrcTypesRebuildSubtypes(result, children.get(0));
- int subtype2 = result.size();
- appendOrcTypesRebuildSubtypes(result, children.get(1));
- type.setKind(OrcProto.Type.Kind.MAP);
- type.addSubtypes(subtype + 1);
- type.addSubtypes(subtype2);
- result.set(subtype, type.build());
- needsAdd = false;
- }
- break;
- case STRUCT:
- {
- List<String> fieldNames = typeDescr.getFieldNames();
-
- // Make room for STRUCT type.
- result.add(null);
-
- List<Integer> fieldSubtypes = new ArrayList<Integer>(fieldNames.size());
- for(TypeDescription child: children) {
- int fieldSubtype = result.size();
- fieldSubtypes.add(fieldSubtype);
- appendOrcTypesRebuildSubtypes(result, child);
- }
-
- type.setKind(OrcProto.Type.Kind.STRUCT);
-
- for (int i = 0 ; i < fieldNames.size(); i++) {
- type.addSubtypes(fieldSubtypes.get(i));
- type.addFieldNames(fieldNames.get(i));
- }
- result.set(subtype, type.build());
- needsAdd = false;
- }
- break;
- case UNION:
- {
- // Make room for UNION type.
- result.add(null);
-
- List<Integer> unionSubtypes = new ArrayList<Integer>(children.size());
- for(TypeDescription child: children) {
- int unionSubtype = result.size();
- unionSubtypes.add(unionSubtype);
- appendOrcTypesRebuildSubtypes(result, child);
- }
-
- type.setKind(OrcProto.Type.Kind.UNION);
- for (int i = 0 ; i < children.size(); i++) {
- type.addSubtypes(unionSubtypes.get(i));
- }
- result.set(subtype, type.build());
- needsAdd = false;
- }
- break;
- default:
- throw new IllegalArgumentException("Unknown category: " + typeDescr.getCategory());
- }
- if (needsAdd) {
- result.add(type.build());
- }
- }
-
- /**
- * NOTE: This method ignores the subtype numbers in the OrcProto.Type rebuilds the subtype
- * numbers based on the length of the result list being appended.
- *
- * @param result
- * @param typeInfo
- */
- public static int appendOrcTypesRebuildSubtypes(List<OrcProto.Type> result,
- List<OrcProto.Type> types, int columnId) {
-
- OrcProto.Type oldType = types.get(columnId++);
-
- int subtype = result.size();
- OrcProto.Type.Builder builder = OrcProto.Type.newBuilder();
- boolean needsAdd = true;
- switch (oldType.getKind()) {
- case BOOLEAN:
- builder.setKind(OrcProto.Type.Kind.BOOLEAN);
- break;
- case BYTE:
- builder.setKind(OrcProto.Type.Kind.BYTE);
- break;
- case SHORT:
- builder.setKind(OrcProto.Type.Kind.SHORT);
- break;
- case INT:
- builder.setKind(OrcProto.Type.Kind.INT);
- break;
- case LONG:
- builder.setKind(OrcProto.Type.Kind.LONG);
- break;
- case FLOAT:
- builder.setKind(OrcProto.Type.Kind.FLOAT);
- break;
- case DOUBLE:
- builder.setKind(OrcProto.Type.Kind.DOUBLE);
- break;
- case STRING:
- builder.setKind(OrcProto.Type.Kind.STRING);
- break;
- case CHAR:
- builder.setKind(OrcProto.Type.Kind.CHAR);
- builder.setMaximumLength(oldType.getMaximumLength());
- break;
- case VARCHAR:
- builder.setKind(Type.Kind.VARCHAR);
- builder.setMaximumLength(oldType.getMaximumLength());
- break;
- case BINARY:
- builder.setKind(OrcProto.Type.Kind.BINARY);
- break;
- case TIMESTAMP:
- builder.setKind(OrcProto.Type.Kind.TIMESTAMP);
- break;
- case DATE:
- builder.setKind(OrcProto.Type.Kind.DATE);
- break;
- case DECIMAL:
- builder.setKind(OrcProto.Type.Kind.DECIMAL);
- builder.setPrecision(oldType.getPrecision());
- builder.setScale(oldType.getScale());
- break;
- case LIST:
- builder.setKind(OrcProto.Type.Kind.LIST);
- builder.addSubtypes(++subtype);
- result.add(builder.build());
- needsAdd = false;
- columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
- break;
- case MAP:
- {
- // Make room for MAP type.
- result.add(null);
-
- // Add MAP type pair in order to determine their subtype values.
- columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
- int subtype2 = result.size();
- columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
- builder.setKind(OrcProto.Type.Kind.MAP);
- builder.addSubtypes(subtype + 1);
- builder.addSubtypes(subtype2);
- result.set(subtype, builder.build());
- needsAdd = false;
- }
- break;
- case STRUCT:
- {
- List<String> fieldNames = oldType.getFieldNamesList();
-
- // Make room for STRUCT type.
- result.add(null);
-
- List<Integer> fieldSubtypes = new ArrayList<Integer>(fieldNames.size());
- for(int i = 0 ; i < fieldNames.size(); i++) {
- int fieldSubtype = result.size();
- fieldSubtypes.add(fieldSubtype);
- columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
- }
-
- builder.setKind(OrcProto.Type.Kind.STRUCT);
-
- for (int i = 0 ; i < fieldNames.size(); i++) {
- builder.addSubtypes(fieldSubtypes.get(i));
- builder.addFieldNames(fieldNames.get(i));
- }
- result.set(subtype, builder.build());
- needsAdd = false;
- }
- break;
- case UNION:
- {
- int subtypeCount = oldType.getSubtypesCount();
-
- // Make room for UNION type.
- result.add(null);
-
- List<Integer> unionSubtypes = new ArrayList<Integer>(subtypeCount);
- for(int i = 0 ; i < subtypeCount; i++) {
- int unionSubtype = result.size();
- unionSubtypes.add(unionSubtype);
- columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
- }
-
- builder.setKind(OrcProto.Type.Kind.UNION);
- for (int i = 0 ; i < subtypeCount; i++) {
- builder.addSubtypes(unionSubtypes.get(i));
- }
- result.set(subtype, builder.build());
- needsAdd = false;
- }
- break;
- default:
- throw new IllegalArgumentException("Unknown category: " + oldType.getKind());
- }
- if (needsAdd) {
- result.add(builder.build());
- }
- return columnId;
- }
-
- public static TypeDescription getDesiredRowTypeDescr(Configuration conf) {
-
- String columnNameProperty = null;
- String columnTypeProperty = null;
-
- ArrayList<String> schemaEvolutionColumnNames = null;
- ArrayList<TypeDescription> schemaEvolutionTypeDescrs = null;
-
- boolean haveSchemaEvolutionProperties = false;
- if (HiveConf.getBoolVar(conf, ConfVars.HIVE_SCHEMA_EVOLUTION)) {
-
- columnNameProperty = conf.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS);
- columnTypeProperty = conf.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES);
-
- haveSchemaEvolutionProperties =
- (columnNameProperty != null && columnTypeProperty != null);
-
- if (haveSchemaEvolutionProperties) {
- schemaEvolutionColumnNames = Lists.newArrayList(columnNameProperty.split(","));
- if (schemaEvolutionColumnNames.size() == 0) {
- haveSchemaEvolutionProperties = false;
- } else {
- schemaEvolutionTypeDescrs =
- OrcUtils.typeDescriptionsFromHiveTypeProperty(columnTypeProperty);
- if (schemaEvolutionTypeDescrs.size() != schemaEvolutionColumnNames.size()) {
- haveSchemaEvolutionProperties = false;
- }
- }
- }
- }
-
- if (!haveSchemaEvolutionProperties) {
-
- // Try regular properties;
- columnNameProperty = conf.get(serdeConstants.LIST_COLUMNS);
- columnTypeProperty = conf.get(serdeConstants.LIST_COLUMN_TYPES);
- if (columnTypeProperty == null || columnNameProperty == null) {
- return null;
- }
-
- schemaEvolutionColumnNames = Lists.newArrayList(columnNameProperty.split(","));
- if (schemaEvolutionColumnNames.size() == 0) {
- return null;
- }
- schemaEvolutionTypeDescrs =
- OrcUtils.typeDescriptionsFromHiveTypeProperty(columnTypeProperty);
- if (schemaEvolutionTypeDescrs.size() != schemaEvolutionColumnNames.size()) {
- return null;
- }
- }
-
- // Desired schema does not include virtual columns or partition columns.
- TypeDescription result = TypeDescription.createStruct();
- for (int i = 0; i < schemaEvolutionColumnNames.size(); i++) {
- result.addField(schemaEvolutionColumnNames.get(i), schemaEvolutionTypeDescrs.get(i));
- }
-
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
deleted file mode 100644
index b228c89..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class OutStream extends PositionedOutputStream {
-
- interface OutputReceiver {
- /**
- * Output the given buffer to the final destination
- * @param buffer the buffer to output
- * @throws IOException
- */
- void output(ByteBuffer buffer) throws IOException;
- }
-
- public static final int HEADER_SIZE = 3;
- private final String name;
- private final OutputReceiver receiver;
- // if enabled the stream will be suppressed when writing stripe
- private boolean suppress;
-
- /**
- * Stores the uncompressed bytes that have been serialized, but not
- * compressed yet. When this fills, we compress the entire buffer.
- */
- private ByteBuffer current = null;
-
- /**
- * Stores the compressed bytes until we have a full buffer and then outputs
- * them to the receiver. If no compression is being done, this (and overflow)
- * will always be null and the current buffer will be sent directly to the
- * receiver.
- */
- private ByteBuffer compressed = null;
-
- /**
- * Since the compressed buffer may start with contents from previous
- * compression blocks, we allocate an overflow buffer so that the
- * output of the codec can be split between the two buffers. After the
- * compressed buffer is sent to the receiver, the overflow buffer becomes
- * the new compressed buffer.
- */
- private ByteBuffer overflow = null;
- private final int bufferSize;
- private final CompressionCodec codec;
- private long compressedBytes = 0;
- private long uncompressedBytes = 0;
-
- OutStream(String name,
- int bufferSize,
- CompressionCodec codec,
- OutputReceiver receiver) throws IOException {
- this.name = name;
- this.bufferSize = bufferSize;
- this.codec = codec;
- this.receiver = receiver;
- this.suppress = false;
- }
-
- public void clear() throws IOException {
- flush();
- suppress = false;
- }
-
- /**
- * Write the length of the compressed bytes. Life is much easier if the
- * header is constant length, so just use 3 bytes. Considering most of the
- * codecs want between 32k (snappy) and 256k (lzo, zlib), 3 bytes should
- * be plenty. We also use the low bit for whether it is the original or
- * compressed bytes.
- * @param buffer the buffer to write the header to
- * @param position the position in the buffer to write at
- * @param val the size in the file
- * @param original is it uncompressed
- */
- private static void writeHeader(ByteBuffer buffer,
- int position,
- int val,
- boolean original) {
- buffer.put(position, (byte) ((val << 1) + (original ? 1 : 0)));
- buffer.put(position + 1, (byte) (val >> 7));
- buffer.put(position + 2, (byte) (val >> 15));
- }
-
- private void getNewInputBuffer() throws IOException {
- if (codec == null) {
- current = ByteBuffer.allocate(bufferSize);
- } else {
- current = ByteBuffer.allocate(bufferSize + HEADER_SIZE);
- writeHeader(current, 0, bufferSize, true);
- current.position(HEADER_SIZE);
- }
- }
-
- /**
- * Allocate a new output buffer if we are compressing.
- */
- private ByteBuffer getNewOutputBuffer() throws IOException {
- return ByteBuffer.allocate(bufferSize + HEADER_SIZE);
- }
-
- private void flip() throws IOException {
- current.limit(current.position());
- current.position(codec == null ? 0 : HEADER_SIZE);
- }
-
- @Override
- public void write(int i) throws IOException {
- if (current == null) {
- getNewInputBuffer();
- }
- if (current.remaining() < 1) {
- spill();
- }
- uncompressedBytes += 1;
- current.put((byte) i);
- }
-
- @Override
- public void write(byte[] bytes, int offset, int length) throws IOException {
- if (current == null) {
- getNewInputBuffer();
- }
- int remaining = Math.min(current.remaining(), length);
- current.put(bytes, offset, remaining);
- uncompressedBytes += remaining;
- length -= remaining;
- while (length != 0) {
- spill();
- offset += remaining;
- remaining = Math.min(current.remaining(), length);
- current.put(bytes, offset, remaining);
- uncompressedBytes += remaining;
- length -= remaining;
- }
- }
-
- private void spill() throws java.io.IOException {
- // if there isn't anything in the current buffer, don't spill
- if (current == null ||
- current.position() == (codec == null ? 0 : HEADER_SIZE)) {
- return;
- }
- flip();
- if (codec == null) {
- receiver.output(current);
- getNewInputBuffer();
- } else {
- if (compressed == null) {
- compressed = getNewOutputBuffer();
- } else if (overflow == null) {
- overflow = getNewOutputBuffer();
- }
- int sizePosn = compressed.position();
- compressed.position(compressed.position() + HEADER_SIZE);
- if (codec.compress(current, compressed, overflow)) {
- uncompressedBytes = 0;
- // move position back to after the header
- current.position(HEADER_SIZE);
- current.limit(current.capacity());
- // find the total bytes in the chunk
- int totalBytes = compressed.position() - sizePosn - HEADER_SIZE;
- if (overflow != null) {
- totalBytes += overflow.position();
- }
- compressedBytes += totalBytes + HEADER_SIZE;
- writeHeader(compressed, sizePosn, totalBytes, false);
- // if we have less than the next header left, spill it.
- if (compressed.remaining() < HEADER_SIZE) {
- compressed.flip();
- receiver.output(compressed);
- compressed = overflow;
- overflow = null;
- }
- } else {
- compressedBytes += uncompressedBytes + HEADER_SIZE;
- uncompressedBytes = 0;
- // we are using the original, but need to spill the current
- // compressed buffer first. So back up to where we started,
- // flip it and add it to done.
- if (sizePosn != 0) {
- compressed.position(sizePosn);
- compressed.flip();
- receiver.output(compressed);
- compressed = null;
- // if we have an overflow, clear it and make it the new compress
- // buffer
- if (overflow != null) {
- overflow.clear();
- compressed = overflow;
- overflow = null;
- }
- } else {
- compressed.clear();
- if (overflow != null) {
- overflow.clear();
- }
- }
-
- // now add the current buffer into the done list and get a new one.
- current.position(0);
- // update the header with the current length
- writeHeader(current, 0, current.limit() - HEADER_SIZE, true);
- receiver.output(current);
- getNewInputBuffer();
- }
- }
- }
-
- void getPosition(PositionRecorder recorder) throws IOException {
- if (codec == null) {
- recorder.addPosition(uncompressedBytes);
- } else {
- recorder.addPosition(compressedBytes);
- recorder.addPosition(uncompressedBytes);
- }
- }
-
- @Override
- public void flush() throws IOException {
- spill();
- if (compressed != null && compressed.position() != 0) {
- compressed.flip();
- receiver.output(compressed);
- compressed = null;
- }
- uncompressedBytes = 0;
- compressedBytes = 0;
- overflow = null;
- current = null;
- }
-
- @Override
- public String toString() {
- return name;
- }
-
- @Override
- public long getBufferSize() {
- long result = 0;
- if (current != null) {
- result += current.capacity();
- }
- if (compressed != null) {
- result += compressed.capacity();
- }
- if (overflow != null) {
- result += overflow.capacity();
- }
- return result;
- }
-
- /**
- * Set suppress flag
- */
- public void suppress() {
- suppress = true;
- }
-
- /**
- * Returns the state of suppress flag
- * @return value of suppress flag
- */
- public boolean isSuppressed() {
- return suppress;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionProvider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionProvider.java
deleted file mode 100644
index 04d81cc..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionProvider.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * An interface used for seeking to a row index.
- */
-public interface PositionProvider {
- long getNext();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionRecorder.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionRecorder.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionRecorder.java
deleted file mode 100644
index 9dc6011..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionRecorder.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * An interface for recording positions in a stream.
- */
-interface PositionRecorder {
- void addPosition(long offset);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionedOutputStream.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionedOutputStream.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionedOutputStream.java
deleted file mode 100644
index 3bee34d..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionedOutputStream.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-abstract class PositionedOutputStream extends OutputStream {
-
- /**
- * Record the current position to the recorder.
- * @param recorder the object that receives the position
- * @throws IOException
- */
- abstract void getPosition(PositionRecorder recorder) throws IOException;
-
- /**
- * Get the memory size currently allocated as buffer associated with this
- * stream.
- * @return the number of bytes used by buffers.
- */
- abstract long getBufferSize();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
index 750cf8d..8823e21 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
@@ -19,10 +19,7 @@
package org.apache.hadoop.hive.ql.io.orc;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -31,78 +28,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
*
* One Reader can support multiple concurrent RecordReader.
*/
-public interface Reader {
-
- /**
- * Get the number of rows in the file.
- * @return the number of rows
- */
- long getNumberOfRows();
-
- /**
- * Get the deserialized data size of the file
- * @return raw data size
- */
- long getRawDataSize();
-
- /**
- * Get the deserialized data size of the specified columns
- * @param colNames
- * @return raw data size of columns
- */
- long getRawDataSizeOfColumns(List<String> colNames);
-
- /**
- * Get the deserialized data size of the specified columns ids
- * @param colIds - internal column id (check orcfiledump for column ids)
- * @return raw data size of columns
- */
- long getRawDataSizeFromColIndices(List<Integer> colIds);
-
- /**
- * Get the user metadata keys.
- * @return the set of metadata keys
- */
- List<String> getMetadataKeys();
-
- /**
- * Get a user metadata value.
- * @param key a key given by the user
- * @return the bytes associated with the given key
- */
- ByteBuffer getMetadataValue(String key);
-
- /**
- * Did the user set the given metadata value.
- * @param key the key to check
- * @return true if the metadata value was set
- */
- boolean hasMetadataValue(String key);
-
- /**
- * Get the compression kind.
- * @return the kind of compression in the file
- */
- CompressionKind getCompression();
-
- /**
- * Get the buffer size for the compression.
- * @return number of bytes to buffer for the compression codec.
- */
- int getCompressionSize();
-
- /**
- * Get the number of rows per a entry in the row index.
- * @return the number of rows per an entry in the row index or 0 if there
- * is no row index.
- */
- int getRowIndexStride();
-
- /**
- * Get the list of stripes.
- * @return the information about the stripes in order
- */
- List<StripeInformation> getStripes();
+public interface Reader extends org.apache.orc.Reader {
/**
* Get the object inspector for looking at the objects.
@@ -111,200 +37,9 @@ public interface Reader {
ObjectInspector getObjectInspector();
/**
- * Get the length of the file.
- * @return the number of bytes in the file
- */
- long getContentLength();
-
- /**
- * Get the statistics about the columns in the file.
- * @return the information about the column
- */
- ColumnStatistics[] getStatistics();
-
- /**
- * Get the list of types contained in the file. The root type is the first
- * type in the list.
- * @return the list of flattened types
- */
- List<OrcProto.Type> getTypes();
-
- /**
- * Get the file format version.
- */
- OrcFile.Version getFileVersion();
-
- /**
- * Get the version of the writer of this file.
+ * Get the Compression kind in the compatibility mode.
*/
- OrcFile.WriterVersion getWriterVersion();
-
- /**
- * Options for creating a RecordReader.
- */
- public static class Options {
- private boolean[] include;
- private long offset = 0;
- private long length = Long.MAX_VALUE;
- private TypeDescription schema;
- private SearchArgument sarg = null;
- private String[] columnNames = null;
- private Boolean useZeroCopy = null;
- private Boolean skipCorruptRecords = null;
-
- /**
- * Set the list of columns to read.
- * @param include a list of columns to read
- * @return this
- */
- public Options include(boolean[] include) {
- this.include = include;
- return this;
- }
-
- /**
- * Set the range of bytes to read
- * @param offset the starting byte offset
- * @param length the number of bytes to read
- * @return this
- */
- public Options range(long offset, long length) {
- this.offset = offset;
- this.length = length;
- return this;
- }
-
- /**
- * Set the schema on read type description.
- */
- public Options schema(TypeDescription schema) {
- this.schema = schema;
- return this;
- }
-
- /**
- * Set search argument for predicate push down.
- * @param sarg the search argument
- * @param columnNames the column names for
- * @return this
- */
- public Options searchArgument(SearchArgument sarg, String[] columnNames) {
- this.sarg = sarg;
- this.columnNames = columnNames;
- return this;
- }
-
- /**
- * Set whether to use zero copy from HDFS.
- * @param value the new zero copy flag
- * @return this
- */
- public Options useZeroCopy(boolean value) {
- this.useZeroCopy = value;
- return this;
- }
-
- /**
- * Set whether to skip corrupt records.
- * @param value the new skip corrupt records flag
- * @return this
- */
- public Options skipCorruptRecords(boolean value) {
- this.skipCorruptRecords = value;
- return this;
- }
-
- public boolean[] getInclude() {
- return include;
- }
-
- public long getOffset() {
- return offset;
- }
-
- public long getLength() {
- return length;
- }
-
- public TypeDescription getSchema() {
- return schema;
- }
-
- public SearchArgument getSearchArgument() {
- return sarg;
- }
-
- public String[] getColumnNames() {
- return columnNames;
- }
-
- public long getMaxOffset() {
- long result = offset + length;
- if (result < 0) {
- result = Long.MAX_VALUE;
- }
- return result;
- }
-
- public Boolean getUseZeroCopy() {
- return useZeroCopy;
- }
-
- public Boolean getSkipCorruptRecords() {
- return skipCorruptRecords;
- }
-
- public Options clone() {
- Options result = new Options();
- result.include = include;
- result.offset = offset;
- result.length = length;
- result.schema = schema;
- result.sarg = sarg;
- result.columnNames = columnNames;
- result.useZeroCopy = useZeroCopy;
- result.skipCorruptRecords = skipCorruptRecords;
- return result;
- }
-
- @Override
- public String toString() {
- StringBuilder buffer = new StringBuilder();
- buffer.append("{include: ");
- if (include == null) {
- buffer.append("null");
- } else {
- buffer.append("[");
- for(int i=0; i < include.length; ++i) {
- if (i != 0) {
- buffer.append(", ");
- }
- buffer.append(include[i]);
- }
- buffer.append("]");
- }
- buffer.append(", offset: ");
- buffer.append(offset);
- buffer.append(", length: ");
- buffer.append(length);
- if (sarg != null) {
- buffer.append(", sarg: ");
- buffer.append(sarg.toString());
- buffer.append(", columns: [");
- for(int i=0; i < columnNames.length; ++i) {
- if (i != 0) {
- buffer.append(", ");
- }
- buffer.append("'");
- buffer.append(columnNames[i]);
- buffer.append("'");
- }
- buffer.append("]");
- }
- buffer.append("}");
- return buffer.toString();
- }
- }
+ CompressionKind getCompression();
/**
* Create a RecordReader that reads everything with the default options.
@@ -314,15 +49,13 @@ public interface Reader {
RecordReader rows() throws IOException;
/**
- * Create a RecordReader that uses the options given.
- * This method can't be named rows, because many callers used rows(null)
- * before the rows() method was introduced.
- * @param options the options to read with
+ * Create a RecordReader that reads everything with the given options.
+ * @param options the options to use
* @return a new RecordReader
* @throws IOException
*/
RecordReader rowsOptions(Options options) throws IOException;
-
+
/**
* Create a RecordReader that will scan the entire file.
* This is a legacy method and rowsOptions is preferred.
@@ -365,44 +98,4 @@ public interface Reader {
boolean[] include, SearchArgument sarg,
String[] neededColumns) throws IOException;
- /**
- * @return Metadata reader used to read file metadata.
- */
- MetadataReader metadata() throws IOException;
-
- /**
- * @return List of integers representing version of the file, in order from major to minor.
- */
- List<Integer> getVersionList();
-
- /**
- * @return Gets the size of metadata, in bytes.
- */
- int getMetadataSize();
-
- /**
- * @return Stripe statistics, in original protobuf form.
- */
- List<OrcProto.StripeStatistics> getOrcProtoStripeStatistics();
-
- /**
- * @return Stripe statistics.
- */
- List<StripeStatistics> getStripeStatistics();
-
- /**
- * @return File statistics, in original protobuf form.
- */
- List<OrcProto.ColumnStatistics> getOrcProtoFileStatistics();
-
- /**
- * @param useZeroCopy Whether zero-copy read should be used.
- * @return The default data reader that ORC is using to read bytes from disk.
- */
- DataReader createDefaultDataReader(boolean useZeroCopy);
-
- /**
- * @return Serialized file metadata read from disk for the purposes of caching, etc.
- */
- ByteBuffer getSerializedFileFooter();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index f2f5f49..e31fd0b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -26,6 +26,18 @@ import java.util.Collections;
import java.util.List;
import java.util.Set;
+import org.apache.orc.impl.BufferChunk;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.impl.ColumnStatisticsImpl;
+import org.apache.orc.CompressionCodec;
+import org.apache.orc.DataReader;
+import org.apache.orc.FileMetaInfo;
+import org.apache.orc.FileMetadata;
+import org.apache.orc.impl.InStream;
+import org.apache.orc.impl.MetadataReader;
+import org.apache.orc.impl.MetadataReaderImpl;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -33,15 +45,12 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.io.DiskRange;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.WriterVersion;
import org.apache.hadoop.hive.ql.io.FileFormatException;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.Text;
+import org.apache.orc.OrcProto;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -55,7 +64,7 @@ public class ReaderImpl implements Reader {
protected final FileSystem fileSystem;
protected final Path path;
- protected final CompressionKind compressionKind;
+ protected final org.apache.orc.CompressionKind compressionKind;
protected final CompressionCodec codec;
protected final int bufferSize;
private final List<OrcProto.StripeStatistics> stripeStats;
@@ -162,7 +171,18 @@ public class ReaderImpl implements Reader {
}
@Override
- public CompressionKind getCompression() {
+ public org.apache.hadoop.hive.ql.io.orc.CompressionKind getCompression() {
+ for (CompressionKind value: org.apache.hadoop.hive.ql.io.orc.CompressionKind.values()) {
+ if (value.getUnderlying() == compressionKind) {
+ return value;
+ }
+ }
+ throw new IllegalArgumentException("Unknown compression kind " +
+ compressionKind);
+ }
+
+ @Override
+ public org.apache.orc.CompressionKind getCompressionKind() {
return compressionKind;
}
@@ -318,7 +338,7 @@ public class ReaderImpl implements Reader {
this.metadataSize = fileMetadata.getMetadataSize();
this.stripeStats = fileMetadata.getStripeStats();
this.versionList = fileMetadata.getVersionList();
- this.writerVersion = WriterVersion.from(fileMetadata.getWriterVersionNum());
+ this.writerVersion = OrcFile.WriterVersion.from(fileMetadata.getWriterVersionNum());
this.types = fileMetadata.getTypes();
this.rowIndexStride = fileMetadata.getRowIndexStride();
this.contentLength = fileMetadata.getContentLength();
@@ -393,7 +413,9 @@ public class ReaderImpl implements Reader {
int footerSize = (int)ps.getFooterLength(), metadataSize = (int)ps.getMetadataLength(),
footerAbsPos = psAbsPos - footerSize, metadataAbsPos = footerAbsPos - metadataSize;
String compressionType = ps.getCompression().toString();
- CompressionCodec codec = WriterImpl.createCodec(CompressionKind.valueOf(compressionType));
+ CompressionCodec codec =
+ WriterImpl.createCodec(org.apache.orc.CompressionKind.valueOf
+ (compressionType));
int bufferSize = (int)ps.getCompressionBlockSize();
bb.position(metadataAbsPos);
bb.mark();
@@ -549,7 +571,7 @@ public class ReaderImpl implements Reader {
*
*/
private static class MetaInfoObjExtractor{
- final CompressionKind compressionKind;
+ final org.apache.orc.CompressionKind compressionKind;
final CompressionCodec codec;
final int bufferSize;
final int metadataSize;
@@ -560,7 +582,7 @@ public class ReaderImpl implements Reader {
MetaInfoObjExtractor(String codecStr, int bufferSize, int metadataSize,
ByteBuffer footerBuffer) throws IOException {
- this.compressionKind = CompressionKind.valueOf(codecStr);
+ this.compressionKind = org.apache.orc.CompressionKind.valueOf(codecStr);
this.bufferSize = bufferSize;
this.codec = WriterImpl.createCodec(compressionKind);
this.metadataSize = metadataSize;
@@ -693,7 +715,7 @@ public class ReaderImpl implements Reader {
List<OrcProto.ColumnStatistics> stats) {
OrcProto.ColumnStatistics colStat = stats.get(colIdx);
long numVals = colStat.getNumberOfValues();
- Type type = types.get(colIdx);
+ OrcProto.Type type = types.get(colIdx);
switch (type.getKind()) {
case BINARY:
@@ -742,7 +764,7 @@ public class ReaderImpl implements Reader {
private List<Integer> getColumnIndicesFromNames(List<String> colNames) {
// top level struct
- Type type = types.get(0);
+ OrcProto.Type type = types.get(0);
List<Integer> colIndices = Lists.newArrayList();
List<String> fieldNames = type.getFieldNamesList();
int fieldIdx = 0;
@@ -789,7 +811,7 @@ public class ReaderImpl implements Reader {
private int getLastIdx() {
Set<Integer> indices = Sets.newHashSet();
- for (Type type : types) {
+ for (OrcProto.Type type : types) {
indices.addAll(type.getSubtypesList());
}
return Collections.max(indices);
@@ -814,7 +836,7 @@ public class ReaderImpl implements Reader {
return result;
}
- public List<UserMetadataItem> getOrcProtoUserMetadata() {
+ public List<OrcProto.UserMetadataItem> getOrcProtoUserMetadata() {
return userMetadata;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java
index dba9071..ff5612d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
/**
* A row-by-row iterator for ORC files.
*/
-public interface RecordReader {
+public interface RecordReader extends org.apache.orc.RecordReader {
/**
* Does the reader have more rows available.
* @return true if there are more rows
@@ -39,40 +39,4 @@ public interface RecordReader {
* @throws java.io.IOException
*/
Object next(Object previous) throws IOException;
-
- /**
- * Read the next row batch. The size of the batch to read cannot be controlled
- * by the callers. Caller need to look at VectorizedRowBatch.size of the retunred
- * object to know the batch size read.
- * @param previousBatch a row batch object that can be reused by the reader
- * @return the row batch that was read
- * @throws java.io.IOException
- */
- VectorizedRowBatch nextBatch(VectorizedRowBatch previousBatch) throws IOException;
-
- /**
- * Get the row number of the row that will be returned by the following
- * call to next().
- * @return the row number from 0 to the number of rows in the file
- * @throws java.io.IOException
- */
- long getRowNumber() throws IOException;
-
- /**
- * Get the progress of the reader through the rows.
- * @return a fraction between 0.0 and 1.0 of rows read
- * @throws java.io.IOException
- */
- float getProgress() throws IOException;
-
- /**
- * Release the resources associated with the given reader.
- * @throws java.io.IOException
- */
- void close() throws IOException;
-
- /**
- * Seek to a particular row number.
- */
- void seekToRow(long rowCount) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
index f36bceb..607003f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.io.orc;
import java.io.IOException;
import java.math.BigDecimal;
-import java.nio.ByteBuffer;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.ArrayList;
@@ -29,6 +28,27 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.orc.BooleanColumnStatistics;
+import org.apache.orc.OrcUtils;
+import org.apache.orc.impl.BufferChunk;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.impl.ColumnStatisticsImpl;
+import org.apache.orc.CompressionCodec;
+import org.apache.orc.DataReader;
+import org.apache.orc.DateColumnStatistics;
+import org.apache.orc.DecimalColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.impl.InStream;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.impl.MetadataReader;
+import org.apache.orc.impl.MetadataReaderImpl;
+import org.apache.orc.OrcConf;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.impl.PositionProvider;
+import org.apache.orc.impl.StreamName;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.TimestampColumnStatistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -41,9 +61,6 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.filters.BloomFilterIO;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
-import org.apache.hadoop.hive.ql.io.orc.TreeReaderFactory.TreeReaderSchema;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
@@ -51,6 +68,7 @@ import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.io.Text;
+import org.apache.orc.OrcProto;
public class RecordReaderImpl implements RecordReader {
static final Logger LOG = LoggerFactory.getLogger(RecordReaderImpl.class);
@@ -84,28 +102,6 @@ public class RecordReaderImpl implements RecordReader {
private final MetadataReader metadata;
private final DataReader dataReader;
- public final static class Index {
- OrcProto.RowIndex[] rowGroupIndex;
- OrcProto.BloomFilterIndex[] bloomFilterIndex;
-
- public Index(OrcProto.RowIndex[] rgIndex, OrcProto.BloomFilterIndex[] bfIndex) {
- this.rowGroupIndex = rgIndex;
- this.bloomFilterIndex = bfIndex;
- }
-
- public OrcProto.RowIndex[] getRowGroupIndex() {
- return rowGroupIndex;
- }
-
- public OrcProto.BloomFilterIndex[] getBloomFilterIndex() {
- return bloomFilterIndex;
- }
-
- public void setRowGroupIndex(OrcProto.RowIndex[] rowGroupIndex) {
- this.rowGroupIndex = rowGroupIndex;
- }
- }
-
/**
* Given a list of column names, find the given column and return the index.
*
@@ -156,15 +152,15 @@ public class RecordReaderImpl implements RecordReader {
Configuration conf
) throws IOException {
- TreeReaderSchema treeReaderSchema;
+ TreeReaderFactory.TreeReaderSchema treeReaderSchema;
if (options.getSchema() == null) {
- treeReaderSchema = new TreeReaderSchema().fileTypes(types).schemaTypes(types);
+ treeReaderSchema = new TreeReaderFactory.TreeReaderSchema().fileTypes(types).schemaTypes(types);
} else {
// Now that we are creating a record reader for a file, validate that the schema to read
// is compatible with the file schema.
//
- List<Type> schemaTypes = OrcUtils.getOrcTypes(options.getSchema());
+ List<OrcProto.Type> schemaTypes = OrcUtils.getOrcTypes(options.getSchema());
treeReaderSchema = SchemaEvolution.validateAndCreate(types, schemaTypes);
}
this.path = path;
@@ -741,7 +737,7 @@ public class RecordReaderImpl implements RecordReader {
if (indexes[columnIx] == null) {
throw new AssertionError("Index is not populated for " + columnIx);
}
- RowIndexEntry entry = indexes[columnIx].getEntry(rowGroup);
+ OrcProto.RowIndexEntry entry = indexes[columnIx].getEntry(rowGroup);
if (entry == null) {
throw new AssertionError("RG is not populated for " + columnIx + " rg " + rowGroup);
}
@@ -801,7 +797,7 @@ public class RecordReaderImpl implements RecordReader {
if (!(range instanceof BufferChunk)) {
continue;
}
- dataReader.releaseBuffer(((BufferChunk) range).chunk);
+ dataReader.releaseBuffer(((BufferChunk) range).getChunk());
}
}
}
@@ -871,60 +867,6 @@ public class RecordReaderImpl implements RecordReader {
}
/**
- * The sections of stripe that we have read.
- * This might not match diskRange - 1 disk range can be multiple buffer chunks, depending on DFS block boundaries.
- */
- public static class BufferChunk extends DiskRangeList {
- final ByteBuffer chunk;
-
- public BufferChunk(ByteBuffer chunk, long offset) {
- super(offset, offset + chunk.remaining());
- this.chunk = chunk;
- }
-
- public ByteBuffer getChunk() {
- return chunk;
- }
-
- @Override
- public boolean hasData() {
- return chunk != null;
- }
-
- @Override
- public final String toString() {
- boolean makesSense = chunk.remaining() == (end - offset);
- return "data range [" + offset + ", " + end + "), size: " + chunk.remaining()
- + (makesSense ? "" : "(!)") + " type: " + (chunk.isDirect() ? "direct" : "array-backed");
- }
-
- @Override
- public DiskRange sliceAndShift(long offset, long end, long shiftBy) {
- assert offset <= end && offset >= this.offset && end <= this.end;
- assert offset + shiftBy >= 0;
- ByteBuffer sliceBuf = chunk.slice();
- int newPos = (int) (offset - this.offset);
- int newLimit = newPos + (int) (end - offset);
- try {
- sliceBuf.position(newPos);
- sliceBuf.limit(newLimit);
- } catch (Throwable t) {
- LOG.error("Failed to slice buffer chunk with range" + " [" + this.offset + ", " + this.end
- + "), position: " + chunk.position() + " limit: " + chunk.limit() + ", "
- + (chunk.isDirect() ? "direct" : "array") + "; to [" + offset + ", " + end + ") "
- + t.getClass());
- throw new RuntimeException(t);
- }
- return new BufferChunk(sliceBuf, offset + shiftBy);
- }
-
- @Override
- public ByteBuffer getData() {
- return chunk;
- }
- }
-
- /**
* Plan the ranges of the file that we need to read given the list of
* columns and row groups.
*
@@ -1199,12 +1141,12 @@ public class RecordReaderImpl implements RecordReader {
throw new IllegalArgumentException("Seek after the end of reader range");
}
- Index readRowIndex(
+ OrcIndex readRowIndex(
int stripeIndex, boolean[] included, boolean[] sargColumns) throws IOException {
return readRowIndex(stripeIndex, included, null, null, sargColumns);
}
- Index readRowIndex(int stripeIndex, boolean[] included, OrcProto.RowIndex[] indexes,
+ OrcIndex readRowIndex(int stripeIndex, boolean[] included, OrcProto.RowIndex[] indexes,
OrcProto.BloomFilterIndex[] bloomFilterIndex, boolean[] sargColumns) throws IOException {
StripeInformation stripe = stripes.get(stripeIndex);
OrcProto.StripeFooter stripeFooter = null;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
index 0caeb1b..8a73948 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.ql.io.orc;
-import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -33,13 +32,18 @@ import org.apache.hadoop.hive.common.io.DiskRange;
import org.apache.hadoop.hive.common.io.DiskRangeList;
import org.apache.hadoop.hive.common.io.DiskRangeList.CreateHelper;
import org.apache.hadoop.hive.common.io.DiskRangeList.MutateHelper;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.HadoopShims.ByteBufferPoolShim;
import org.apache.hadoop.hive.shims.HadoopShims.ZeroCopyReaderShim;
+import org.apache.orc.impl.BufferChunk;
+import org.apache.orc.CompressionCodec;
+import org.apache.orc.DataReader;
+import org.apache.orc.impl.DirectDecompressionCodec;
+import org.apache.orc.OrcProto;
import com.google.common.collect.ComparisonChain;
+import org.apache.orc.impl.OutStream;
/**
* Stateless methods shared between RecordReaderImpl and EncodedReaderImpl.
@@ -262,7 +266,7 @@ public class RecordReaderUtils {
/**
* Build a string representation of a list of disk ranges.
- * @param ranges ranges to stringify
+ * @param range ranges to stringify
* @return the resulting string
*/
public static String stringifyDiskRanges(DiskRangeList range) {
@@ -288,7 +292,7 @@ public class RecordReaderUtils {
* Read the list of ranges from the file.
* @param file the file to read
* @param base the base of the stripe
- * @param ranges the disk ranges within the stripe to read
+ * @param range the disk ranges within the stripe to read
* @return the bytes read for each disk range, which is the same length as
* ranges
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java
deleted file mode 100644
index d98713c..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * A memory efficient red-black tree that does not allocate any objects per
- * an element. This class is abstract and assumes that the child class
- * handles the key and comparisons with the key.
- */
-abstract class RedBlackTree {
- public static final int NULL = -1;
-
- // Various values controlling the offset of the data within the array.
- private static final int LEFT_OFFSET = 0;
- private static final int RIGHT_OFFSET = 1;
- private static final int ELEMENT_SIZE = 2;
-
- protected int size = 0;
- private final DynamicIntArray data;
- protected int root = NULL;
- protected int lastAdd = 0;
- private boolean wasAdd = false;
-
- /**
- * Create a set with the given initial capacity.
- */
- public RedBlackTree(int initialCapacity) {
- data = new DynamicIntArray(initialCapacity * ELEMENT_SIZE);
- }
-
- /**
- * Insert a new node into the data array, growing the array as necessary.
- *
- * @return Returns the position of the new node.
- */
- private int insert(int left, int right, boolean isRed) {
- int position = size;
- size += 1;
- setLeft(position, left, isRed);
- setRight(position, right);
- return position;
- }
-
- /**
- * Compare the value at the given position to the new value.
- * @return 0 if the values are the same, -1 if the new value is smaller and
- * 1 if the new value is larger.
- */
- protected abstract int compareValue(int position);
-
- /**
- * Is the given node red as opposed to black? To prevent having an extra word
- * in the data array, we just the low bit on the left child index.
- */
- protected boolean isRed(int position) {
- return position != NULL &&
- (data.get(position * ELEMENT_SIZE + LEFT_OFFSET) & 1) == 1;
- }
-
- /**
- * Set the red bit true or false.
- */
- private void setRed(int position, boolean isRed) {
- int offset = position * ELEMENT_SIZE + LEFT_OFFSET;
- if (isRed) {
- data.set(offset, data.get(offset) | 1);
- } else {
- data.set(offset, data.get(offset) & ~1);
- }
- }
-
- /**
- * Get the left field of the given position.
- */
- protected int getLeft(int position) {
- return data.get(position * ELEMENT_SIZE + LEFT_OFFSET) >> 1;
- }
-
- /**
- * Get the right field of the given position.
- */
- protected int getRight(int position) {
- return data.get(position * ELEMENT_SIZE + RIGHT_OFFSET);
- }
-
- /**
- * Set the left field of the given position.
- * Note that we are storing the node color in the low bit of the left pointer.
- */
- private void setLeft(int position, int left) {
- int offset = position * ELEMENT_SIZE + LEFT_OFFSET;
- data.set(offset, (left << 1) | (data.get(offset) & 1));
- }
-
- /**
- * Set the left field of the given position.
- * Note that we are storing the node color in the low bit of the left pointer.
- */
- private void setLeft(int position, int left, boolean isRed) {
- int offset = position * ELEMENT_SIZE + LEFT_OFFSET;
- data.set(offset, (left << 1) | (isRed ? 1 : 0));
- }
-
- /**
- * Set the right field of the given position.
- */
- private void setRight(int position, int right) {
- data.set(position * ELEMENT_SIZE + RIGHT_OFFSET, right);
- }
-
- /**
- * Insert or find a given key in the tree and rebalance the tree correctly.
- * Rebalancing restores the red-black aspect of the tree to maintain the
- * invariants:
- * 1. If a node is red, both of its children are black.
- * 2. Each child of a node has the same black height (the number of black
- * nodes between it and the leaves of the tree).
- *
- * Inserted nodes are at the leaves and are red, therefore there is at most a
- * violation of rule 1 at the node we just put in. Instead of always keeping
- * the parents, this routine passing down the context.
- *
- * The fix is broken down into 6 cases (1.{1,2,3} and 2.{1,2,3} that are
- * left-right mirror images of each other). See Algorighms by Cormen,
- * Leiserson, and Rivest for the explaination of the subcases.
- *
- * @param node The node that we are fixing right now.
- * @param fromLeft Did we come down from the left?
- * @param parent Nodes' parent
- * @param grandparent Parent's parent
- * @param greatGrandparent Grandparent's parent
- * @return Does parent also need to be checked and/or fixed?
- */
- private boolean add(int node, boolean fromLeft, int parent,
- int grandparent, int greatGrandparent) {
- if (node == NULL) {
- if (root == NULL) {
- lastAdd = insert(NULL, NULL, false);
- root = lastAdd;
- wasAdd = true;
- return false;
- } else {
- lastAdd = insert(NULL, NULL, true);
- node = lastAdd;
- wasAdd = true;
- // connect the new node into the tree
- if (fromLeft) {
- setLeft(parent, node);
- } else {
- setRight(parent, node);
- }
- }
- } else {
- int compare = compareValue(node);
- boolean keepGoing;
-
- // Recurse down to find where the node needs to be added
- if (compare < 0) {
- keepGoing = add(getLeft(node), true, node, parent, grandparent);
- } else if (compare > 0) {
- keepGoing = add(getRight(node), false, node, parent, grandparent);
- } else {
- lastAdd = node;
- wasAdd = false;
- return false;
- }
-
- // we don't need to fix the root (because it is always set to black)
- if (node == root || !keepGoing) {
- return false;
- }
- }
-
-
- // Do we need to fix this node? Only if there are two reds right under each
- // other.
- if (isRed(node) && isRed(parent)) {
- if (parent == getLeft(grandparent)) {
- int uncle = getRight(grandparent);
- if (isRed(uncle)) {
- // case 1.1
- setRed(parent, false);
- setRed(uncle, false);
- setRed(grandparent, true);
- return true;
- } else {
- if (node == getRight(parent)) {
- // case 1.2
- // swap node and parent
- int tmp = node;
- node = parent;
- parent = tmp;
- // left-rotate on node
- setLeft(grandparent, parent);
- setRight(node, getLeft(parent));
- setLeft(parent, node);
- }
-
- // case 1.2 and 1.3
- setRed(parent, false);
- setRed(grandparent, true);
-
- // right-rotate on grandparent
- if (greatGrandparent == NULL) {
- root = parent;
- } else if (getLeft(greatGrandparent) == grandparent) {
- setLeft(greatGrandparent, parent);
- } else {
- setRight(greatGrandparent, parent);
- }
- setLeft(grandparent, getRight(parent));
- setRight(parent, grandparent);
- return false;
- }
- } else {
- int uncle = getLeft(grandparent);
- if (isRed(uncle)) {
- // case 2.1
- setRed(parent, false);
- setRed(uncle, false);
- setRed(grandparent, true);
- return true;
- } else {
- if (node == getLeft(parent)) {
- // case 2.2
- // swap node and parent
- int tmp = node;
- node = parent;
- parent = tmp;
- // right-rotate on node
- setRight(grandparent, parent);
- setLeft(node, getRight(parent));
- setRight(parent, node);
- }
- // case 2.2 and 2.3
- setRed(parent, false);
- setRed(grandparent, true);
- // left-rotate on grandparent
- if (greatGrandparent == NULL) {
- root = parent;
- } else if (getRight(greatGrandparent) == grandparent) {
- setRight(greatGrandparent, parent);
- } else {
- setLeft(greatGrandparent, parent);
- }
- setRight(grandparent, getLeft(parent));
- setLeft(parent, grandparent);
- return false;
- }
- }
- } else {
- return true;
- }
- }
-
- /**
- * Add the new key to the tree.
- * @return true if the element is a new one.
- */
- protected boolean add() {
- add(root, false, NULL, NULL, NULL);
- if (wasAdd) {
- setRed(root, false);
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Get the number of elements in the set.
- */
- public int size() {
- return size;
- }
-
- /**
- * Reset the table to empty.
- */
- public void clear() {
- root = NULL;
- size = 0;
- data.clear();
- }
-
- /**
- * Get the buffer size in bytes.
- */
- public long getSizeInBytes() {
- return data.getSizeInBytes();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java
deleted file mode 100644
index f3e6184..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-
-/**
- * A reader that reads a sequence of bytes. A control byte is read before
- * each run with positive values 0 to 127 meaning 3 to 130 repetitions. If the
- * byte is -1 to -128, 1 to 128 literal byte values follow.
- */
-public class RunLengthByteReader {
- private InStream input;
- private final byte[] literals =
- new byte[RunLengthByteWriter.MAX_LITERAL_SIZE];
- private int numLiterals = 0;
- private int used = 0;
- private boolean repeat = false;
-
- RunLengthByteReader(InStream input) throws IOException {
- this.input = input;
- }
-
- public void setInStream(InStream input) {
- this.input = input;
- }
-
- private void readValues(boolean ignoreEof) throws IOException {
- int control = input.read();
- used = 0;
- if (control == -1) {
- if (!ignoreEof) {
- throw new EOFException("Read past end of buffer RLE byte from " + input);
- }
- used = numLiterals = 0;
- return;
- } else if (control < 0x80) {
- repeat = true;
- numLiterals = control + RunLengthByteWriter.MIN_REPEAT_SIZE;
- int val = input.read();
- if (val == -1) {
- throw new EOFException("Reading RLE byte got EOF");
- }
- literals[0] = (byte) val;
- } else {
- repeat = false;
- numLiterals = 0x100 - control;
- int bytes = 0;
- while (bytes < numLiterals) {
- int result = input.read(literals, bytes, numLiterals - bytes);
- if (result == -1) {
- throw new EOFException("Reading RLE byte literal got EOF in " + this);
- }
- bytes += result;
- }
- }
- }
-
- boolean hasNext() throws IOException {
- return used != numLiterals || input.available() > 0;
- }
-
- byte next() throws IOException {
- byte result;
- if (used == numLiterals) {
- readValues(false);
- }
- if (repeat) {
- result = literals[0];
- } else {
- result = literals[used];
- }
- ++used;
- return result;
- }
-
- void nextVector(LongColumnVector previous, long previousLen)
- throws IOException {
- previous.isRepeating = true;
- for (int i = 0; i < previousLen; i++) {
- if (!previous.isNull[i]) {
- previous.vector[i] = next();
- } else {
- // The default value of null for int types in vectorized
- // processing is 1, so set that if the value is null
- previous.vector[i] = 1;
- }
-
- // The default value for nulls in Vectorization for int types is 1
- // and given that non null value can also be 1, we need to check for isNull also
- // when determining the isRepeating flag.
- if (previous.isRepeating
- && i > 0
- && ((previous.vector[i - 1] != previous.vector[i]) || (previous.isNull[i - 1] != previous.isNull[i]))) {
- previous.isRepeating = false;
- }
- }
- }
-
- public void seek(PositionProvider index) throws IOException {
- input.seek(index);
- int consumed = (int) index.getNext();
- if (consumed != 0) {
- // a loop is required for cases where we break the run into two parts
- while (consumed > 0) {
- readValues(false);
- used = consumed;
- consumed -= numLiterals;
- }
- } else {
- used = 0;
- numLiterals = 0;
- }
- }
-
- void skip(long items) throws IOException {
- while (items > 0) {
- if (used == numLiterals) {
- readValues(false);
- }
- long consume = Math.min(items, numLiterals - used);
- used += consume;
- items -= consume;
- }
- }
-
- @Override
- public String toString() {
- return "byte rle " + (repeat ? "repeat" : "literal") + " used: " +
- used + "/" + numLiterals + " from " + input;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java
deleted file mode 100644
index ab4bbcc..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-
-/**
- * A streamFactory that writes a sequence of bytes. A control byte is written before
- * each run with positive values 0 to 127 meaning 2 to 129 repetitions. If the
- * bytes is -1 to -128, 1 to 128 literal byte values follow.
- */
-class RunLengthByteWriter {
- static final int MIN_REPEAT_SIZE = 3;
- static final int MAX_LITERAL_SIZE = 128;
- static final int MAX_REPEAT_SIZE= 127 + MIN_REPEAT_SIZE;
- private final PositionedOutputStream output;
- private final byte[] literals = new byte[MAX_LITERAL_SIZE];
- private int numLiterals = 0;
- private boolean repeat = false;
- private int tailRunLength = 0;
-
- RunLengthByteWriter(PositionedOutputStream output) {
- this.output = output;
- }
-
- private void writeValues() throws IOException {
- if (numLiterals != 0) {
- if (repeat) {
- output.write(numLiterals - MIN_REPEAT_SIZE);
- output.write(literals, 0, 1);
- } else {
- output.write(-numLiterals);
- output.write(literals, 0, numLiterals);
- }
- repeat = false;
- tailRunLength = 0;
- numLiterals = 0;
- }
- }
-
- void flush() throws IOException {
- writeValues();
- output.flush();
- }
-
- void write(byte value) throws IOException {
- if (numLiterals == 0) {
- literals[numLiterals++] = value;
- tailRunLength = 1;
- } else if (repeat) {
- if (value == literals[0]) {
- numLiterals += 1;
- if (numLiterals == MAX_REPEAT_SIZE) {
- writeValues();
- }
- } else {
- writeValues();
- literals[numLiterals++] = value;
- tailRunLength = 1;
- }
- } else {
- if (value == literals[numLiterals - 1]) {
- tailRunLength += 1;
- } else {
- tailRunLength = 1;
- }
- if (tailRunLength == MIN_REPEAT_SIZE) {
- if (numLiterals + 1 == MIN_REPEAT_SIZE) {
- repeat = true;
- numLiterals += 1;
- } else {
- numLiterals -= MIN_REPEAT_SIZE - 1;
- writeValues();
- literals[0] = value;
- repeat = true;
- numLiterals = MIN_REPEAT_SIZE;
- }
- } else {
- literals[numLiterals++] = value;
- if (numLiterals == MAX_LITERAL_SIZE) {
- writeValues();
- }
- }
- }
- }
-
- void getPosition(PositionRecorder recorder) throws IOException {
- output.getPosition(recorder);
- recorder.addPosition(numLiterals);
- }
-}
[12/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/InStream.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/InStream.java b/orc/src/java/org/apache/orc/impl/InStream.java
new file mode 100644
index 0000000..b1c6de5
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/InStream.java
@@ -0,0 +1,496 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+
+import org.apache.orc.CompressionCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.common.io.DiskRange;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.CodedInputStream;
+
+public abstract class InStream extends InputStream {
+
+ private static final Logger LOG = LoggerFactory.getLogger(InStream.class);
+ private static final int PROTOBUF_MESSAGE_MAX_LIMIT = 1024 << 20; // 1GB
+
+ protected final String name;
+ protected long length;
+
+ public InStream(String name, long length) {
+ this.name = name;
+ this.length = length;
+ }
+
+ public String getStreamName() {
+ return name;
+ }
+
+ public long getStreamLength() {
+ return length;
+ }
+
+ public static class UncompressedStream extends InStream {
+ private List<DiskRange> bytes;
+ private long length;
+ protected long currentOffset;
+ private ByteBuffer range;
+ private int currentRange;
+
+ public UncompressedStream(String name, List<DiskRange> input, long length) {
+ super(name, length);
+ reset(input, length);
+ }
+
+ protected void reset(List<DiskRange> input, long length) {
+ this.bytes = input;
+ this.length = length;
+ currentRange = 0;
+ currentOffset = 0;
+ range = null;
+ }
+
+ @Override
+ public int read() {
+ if (range == null || range.remaining() == 0) {
+ if (currentOffset == length) {
+ return -1;
+ }
+ seek(currentOffset);
+ }
+ currentOffset += 1;
+ return 0xff & range.get();
+ }
+
+ @Override
+ public int read(byte[] data, int offset, int length) {
+ if (range == null || range.remaining() == 0) {
+ if (currentOffset == this.length) {
+ return -1;
+ }
+ seek(currentOffset);
+ }
+ int actualLength = Math.min(length, range.remaining());
+ range.get(data, offset, actualLength);
+ currentOffset += actualLength;
+ return actualLength;
+ }
+
+ @Override
+ public int available() {
+ if (range != null && range.remaining() > 0) {
+ return range.remaining();
+ }
+ return (int) (length - currentOffset);
+ }
+
+ @Override
+ public void close() {
+ currentRange = bytes.size();
+ currentOffset = length;
+ // explicit de-ref of bytes[]
+ bytes.clear();
+ }
+
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ seek(index.getNext());
+ }
+
+ public void seek(long desired) {
+ if (desired == 0 && bytes.isEmpty()) {
+ logEmptySeek(name);
+ return;
+ }
+ int i = 0;
+ for (DiskRange curRange : bytes) {
+ if (desired == 0 && curRange.getData().remaining() == 0) {
+ logEmptySeek(name);
+ return;
+ }
+ if (curRange.getOffset() <= desired &&
+ (desired - curRange.getOffset()) < curRange.getLength()) {
+ currentOffset = desired;
+ currentRange = i;
+ this.range = curRange.getData().duplicate();
+ int pos = range.position();
+ pos += (int)(desired - curRange.getOffset()); // this is why we duplicate
+ this.range.position(pos);
+ return;
+ }
+ ++i;
+ }
+ // if they are seeking to the precise end, go ahead and let them go there
+ int segments = bytes.size();
+ if (segments != 0 && desired == bytes.get(segments - 1).getEnd()) {
+ currentOffset = desired;
+ currentRange = segments - 1;
+ DiskRange curRange = bytes.get(currentRange);
+ this.range = curRange.getData().duplicate();
+ int pos = range.position();
+ pos += (int)(desired - curRange.getOffset()); // this is why we duplicate
+ this.range.position(pos);
+ return;
+ }
+ throw new IllegalArgumentException("Seek in " + name + " to " +
+ desired + " is outside of the data");
+ }
+
+ @Override
+ public String toString() {
+ return "uncompressed stream " + name + " position: " + currentOffset +
+ " length: " + length + " range: " + currentRange +
+ " offset: " + (range == null ? 0 : range.position()) + " limit: " + (range == null ? 0 : range.limit());
+ }
+ }
+
+ private static ByteBuffer allocateBuffer(int size, boolean isDirect) {
+ // TODO: use the same pool as the ORC readers
+ if (isDirect) {
+ return ByteBuffer.allocateDirect(size);
+ } else {
+ return ByteBuffer.allocate(size);
+ }
+ }
+
+ private static class CompressedStream extends InStream {
+ private final List<DiskRange> bytes;
+ private final int bufferSize;
+ private ByteBuffer uncompressed;
+ private final CompressionCodec codec;
+ private ByteBuffer compressed;
+ private long currentOffset;
+ private int currentRange;
+ private boolean isUncompressedOriginal;
+
+ public CompressedStream(String name, List<DiskRange> input, long length,
+ CompressionCodec codec, int bufferSize) {
+ super(name, length);
+ this.bytes = input;
+ this.codec = codec;
+ this.bufferSize = bufferSize;
+ currentOffset = 0;
+ currentRange = 0;
+ }
+
+ private void allocateForUncompressed(int size, boolean isDirect) {
+ uncompressed = allocateBuffer(size, isDirect);
+ }
+
+ private void readHeader() throws IOException {
+ if (compressed == null || compressed.remaining() <= 0) {
+ seek(currentOffset);
+ }
+ if (compressed.remaining() > OutStream.HEADER_SIZE) {
+ int b0 = compressed.get() & 0xff;
+ int b1 = compressed.get() & 0xff;
+ int b2 = compressed.get() & 0xff;
+ boolean isOriginal = (b0 & 0x01) == 1;
+ int chunkLength = (b2 << 15) | (b1 << 7) | (b0 >> 1);
+
+ if (chunkLength > bufferSize) {
+ throw new IllegalArgumentException("Buffer size too small. size = " +
+ bufferSize + " needed = " + chunkLength);
+ }
+ // read 3 bytes, which should be equal to OutStream.HEADER_SIZE always
+ assert OutStream.HEADER_SIZE == 3 : "The Orc HEADER_SIZE must be the same in OutStream and InStream";
+ currentOffset += OutStream.HEADER_SIZE;
+
+ ByteBuffer slice = this.slice(chunkLength);
+
+ if (isOriginal) {
+ uncompressed = slice;
+ isUncompressedOriginal = true;
+ } else {
+ if (isUncompressedOriginal) {
+ allocateForUncompressed(bufferSize, slice.isDirect());
+ isUncompressedOriginal = false;
+ } else if (uncompressed == null) {
+ allocateForUncompressed(bufferSize, slice.isDirect());
+ } else {
+ uncompressed.clear();
+ }
+ codec.decompress(slice, uncompressed);
+ }
+ } else {
+ throw new IllegalStateException("Can't read header at " + this);
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (uncompressed == null || uncompressed.remaining() == 0) {
+ if (currentOffset == length) {
+ return -1;
+ }
+ readHeader();
+ }
+ return 0xff & uncompressed.get();
+ }
+
+ @Override
+ public int read(byte[] data, int offset, int length) throws IOException {
+ if (uncompressed == null || uncompressed.remaining() == 0) {
+ if (currentOffset == this.length) {
+ return -1;
+ }
+ readHeader();
+ }
+ int actualLength = Math.min(length, uncompressed.remaining());
+ uncompressed.get(data, offset, actualLength);
+ return actualLength;
+ }
+
+ @Override
+ public int available() throws IOException {
+ if (uncompressed == null || uncompressed.remaining() == 0) {
+ if (currentOffset == length) {
+ return 0;
+ }
+ readHeader();
+ }
+ return uncompressed.remaining();
+ }
+
+ @Override
+ public void close() {
+ uncompressed = null;
+ compressed = null;
+ currentRange = bytes.size();
+ currentOffset = length;
+ bytes.clear();
+ }
+
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ seek(index.getNext());
+ long uncompressedBytes = index.getNext();
+ if (uncompressedBytes != 0) {
+ readHeader();
+ uncompressed.position(uncompressed.position() +
+ (int) uncompressedBytes);
+ } else if (uncompressed != null) {
+ // mark the uncompressed buffer as done
+ uncompressed.position(uncompressed.limit());
+ }
+ }
+
+ /* slices a read only contiguous buffer of chunkLength */
+ private ByteBuffer slice(int chunkLength) throws IOException {
+ int len = chunkLength;
+ final long oldOffset = currentOffset;
+ ByteBuffer slice;
+ if (compressed.remaining() >= len) {
+ slice = compressed.slice();
+ // simple case
+ slice.limit(len);
+ currentOffset += len;
+ compressed.position(compressed.position() + len);
+ return slice;
+ } else if (currentRange >= (bytes.size() - 1)) {
+ // nothing has been modified yet
+ throw new IOException("EOF in " + this + " while trying to read " +
+ chunkLength + " bytes");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format(
+ "Crossing into next BufferChunk because compressed only has %d bytes (needs %d)",
+ compressed.remaining(), len));
+ }
+
+ // we need to consolidate 2 or more buffers into 1
+ // first copy out compressed buffers
+ ByteBuffer copy = allocateBuffer(chunkLength, compressed.isDirect());
+ currentOffset += compressed.remaining();
+ len -= compressed.remaining();
+ copy.put(compressed);
+ ListIterator<DiskRange> iter = bytes.listIterator(currentRange);
+
+ while (len > 0 && iter.hasNext()) {
+ ++currentRange;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Read slow-path, >1 cross block reads with %s", this.toString()));
+ }
+ DiskRange range = iter.next();
+ compressed = range.getData().duplicate();
+ if (compressed.remaining() >= len) {
+ slice = compressed.slice();
+ slice.limit(len);
+ copy.put(slice);
+ currentOffset += len;
+ compressed.position(compressed.position() + len);
+ return copy;
+ }
+ currentOffset += compressed.remaining();
+ len -= compressed.remaining();
+ copy.put(compressed);
+ }
+
+ // restore offsets for exception clarity
+ seek(oldOffset);
+ throw new IOException("EOF in " + this + " while trying to read " +
+ chunkLength + " bytes");
+ }
+
+ private void seek(long desired) throws IOException {
+ if (desired == 0 && bytes.isEmpty()) {
+ logEmptySeek(name);
+ return;
+ }
+ int i = 0;
+ for (DiskRange range : bytes) {
+ if (range.getOffset() <= desired && desired < range.getEnd()) {
+ currentRange = i;
+ compressed = range.getData().duplicate();
+ int pos = compressed.position();
+ pos += (int)(desired - range.getOffset());
+ compressed.position(pos);
+ currentOffset = desired;
+ return;
+ }
+ ++i;
+ }
+ // if they are seeking to the precise end, go ahead and let them go there
+ int segments = bytes.size();
+ if (segments != 0 && desired == bytes.get(segments - 1).getEnd()) {
+ DiskRange range = bytes.get(segments - 1);
+ currentRange = segments - 1;
+ compressed = range.getData().duplicate();
+ compressed.position(compressed.limit());
+ currentOffset = desired;
+ return;
+ }
+ throw new IOException("Seek outside of data in " + this + " to " + desired);
+ }
+
+ private String rangeString() {
+ StringBuilder builder = new StringBuilder();
+ int i = 0;
+ for (DiskRange range : bytes) {
+ if (i != 0) {
+ builder.append("; ");
+ }
+ builder.append(" range " + i + " = " + range.getOffset()
+ + " to " + (range.getEnd() - range.getOffset()));
+ ++i;
+ }
+ return builder.toString();
+ }
+
+ @Override
+ public String toString() {
+ return "compressed stream " + name + " position: " + currentOffset +
+ " length: " + length + " range: " + currentRange +
+ " offset: " + (compressed == null ? 0 : compressed.position()) + " limit: " + (compressed == null ? 0 : compressed.limit()) +
+ rangeString() +
+ (uncompressed == null ? "" :
+ " uncompressed: " + uncompressed.position() + " to " +
+ uncompressed.limit());
+ }
+ }
+
+ public abstract void seek(PositionProvider index) throws IOException;
+
+ private static void logEmptySeek(String name) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn("Attempting seek into empty stream (" + name + ") Skipping stream.");
+ }
+ }
+
+ /**
+ * Create an input stream from a list of buffers.
+ * @param fileName name of the file
+ * @param streamName the name of the stream
+ * @param buffers the list of ranges of bytes for the stream
+ * @param offsets a list of offsets (the same length as input) that must
+ * contain the first offset of the each set of bytes in input
+ * @param length the length in bytes of the stream
+ * @param codec the compression codec
+ * @param bufferSize the compression buffer size
+ * @return an input stream
+ * @throws IOException
+ */
+ @VisibleForTesting
+ @Deprecated
+ public static InStream create(String streamName,
+ ByteBuffer[] buffers,
+ long[] offsets,
+ long length,
+ CompressionCodec codec,
+ int bufferSize) throws IOException {
+ List<DiskRange> input = new ArrayList<DiskRange>(buffers.length);
+ for (int i = 0; i < buffers.length; ++i) {
+ input.add(new BufferChunk(buffers[i], offsets[i]));
+ }
+ return create(streamName, input, length, codec, bufferSize);
+ }
+
+ /**
+ * Create an input stream from a list of disk ranges with data.
+ * @param name the name of the stream
+ * @param input the list of ranges of bytes for the stream; from disk or cache
+ * @param length the length in bytes of the stream
+ * @param codec the compression codec
+ * @param bufferSize the compression buffer size
+ * @return an input stream
+ * @throws IOException
+ */
+ public static InStream create(String name,
+ List<DiskRange> input,
+ long length,
+ CompressionCodec codec,
+ int bufferSize) throws IOException {
+ if (codec == null) {
+ return new UncompressedStream(name, input, length);
+ } else {
+ return new CompressedStream(name, input, length, codec, bufferSize);
+ }
+ }
+
+ /**
+ * Creates coded input stream (used for protobuf message parsing) with higher message size limit.
+ *
+ * @param name the name of the stream
+ * @param input the list of ranges of bytes for the stream; from disk or cache
+ * @param length the length in bytes of the stream
+ * @param codec the compression codec
+ * @param bufferSize the compression buffer size
+ * @return coded input stream
+ * @throws IOException
+ */
+ public static CodedInputStream createCodedInputStream(
+ String name,
+ List<DiskRange> input,
+ long length,
+ CompressionCodec codec,
+ int bufferSize) throws IOException {
+ InStream inStream = create(name, input, length, codec, bufferSize);
+ CodedInputStream codedInputStream = CodedInputStream.newInstance(inStream);
+ codedInputStream.setSizeLimit(PROTOBUF_MESSAGE_MAX_LIMIT);
+ return codedInputStream;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/IntegerReader.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/IntegerReader.java b/orc/src/java/org/apache/orc/impl/IntegerReader.java
new file mode 100644
index 0000000..b928559
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/IntegerReader.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+
+/**
+ * Interface for reading integers.
+ */
+public interface IntegerReader {
+
+ /**
+ * Seek to the position provided by index.
+ * @param index
+ * @throws IOException
+ */
+ void seek(PositionProvider index) throws IOException;
+
+ /**
+ * Skip number of specified rows.
+ * @param numValues
+ * @throws IOException
+ */
+ void skip(long numValues) throws IOException;
+
+ /**
+ * Check if there are any more values left.
+ * @return
+ * @throws IOException
+ */
+ boolean hasNext() throws IOException;
+
+ /**
+ * Return the next available value.
+ * @return
+ * @throws IOException
+ */
+ long next() throws IOException;
+
+ /**
+ * Return the next available vector for values.
+ * @return
+ * @throws IOException
+ */
+ void nextVector(LongColumnVector previous, long previousLen)
+ throws IOException;
+
+ void setInStream(InStream data);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/IntegerWriter.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/IntegerWriter.java b/orc/src/java/org/apache/orc/impl/IntegerWriter.java
new file mode 100644
index 0000000..419054f
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/IntegerWriter.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import java.io.IOException;
+
+/**
+ * Interface for writing integers.
+ */
+public interface IntegerWriter {
+
+ /**
+ * Get position from the stream.
+ * @param recorder
+ * @throws IOException
+ */
+ void getPosition(PositionRecorder recorder) throws IOException;
+
+ /**
+ * Write the integer value
+ * @param value
+ * @throws IOException
+ */
+ void write(long value) throws IOException;
+
+ /**
+ * Flush the buffer
+ * @throws IOException
+ */
+ void flush() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/MemoryManager.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/MemoryManager.java b/orc/src/java/org/apache/orc/impl/MemoryManager.java
new file mode 100644
index 0000000..2dbfba7
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/MemoryManager.java
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.orc.OrcConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import com.google.common.base.Preconditions;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Implements a memory manager that keeps a global context of how many ORC
+ * writers there are and manages the memory between them. For use cases with
+ * dynamic partitions, it is easy to end up with many writers in the same task.
+ * By managing the size of each allocation, we try to cut down the size of each
+ * allocation and keep the task from running out of memory.
+ *
+ * This class is not thread safe, but is re-entrant - ensure creation and all
+ * invocations are triggered from the same thread.
+ */
+public class MemoryManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MemoryManager.class);
+
+ /**
+ * How often should we check the memory sizes? Measured in rows added
+ * to all of the writers.
+ */
+ private static final int ROWS_BETWEEN_CHECKS = 5000;
+ private final long totalMemoryPool;
+ private final Map<Path, WriterInfo> writerList =
+ new HashMap<Path, WriterInfo>();
+ private long totalAllocation = 0;
+ private double currentScale = 1;
+ private int rowsAddedSinceCheck = 0;
+ private final OwnedLock ownerLock = new OwnedLock();
+
+ @SuppressWarnings("serial")
+ private static class OwnedLock extends ReentrantLock {
+ public Thread getOwner() {
+ return super.getOwner();
+ }
+ }
+
+ private static class WriterInfo {
+ long allocation;
+ Callback callback;
+ WriterInfo(long allocation, Callback callback) {
+ this.allocation = allocation;
+ this.callback = callback;
+ }
+ }
+
+ public interface Callback {
+ /**
+ * The writer needs to check its memory usage
+ * @param newScale the current scale factor for memory allocations
+ * @return true if the writer was over the limit
+ * @throws IOException
+ */
+ boolean checkMemory(double newScale) throws IOException;
+ }
+
+ /**
+ * Create the memory manager.
+ * @param conf use the configuration to find the maximum size of the memory
+ * pool.
+ */
+ public MemoryManager(Configuration conf) {
+ double maxLoad = OrcConf.MEMORY_POOL.getDouble(conf);
+ totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().
+ getHeapMemoryUsage().getMax() * maxLoad);
+ ownerLock.lock();
+ }
+
+ /**
+ * Light weight thread-safety check for multi-threaded access patterns
+ */
+ private void checkOwner() {
+ Preconditions.checkArgument(ownerLock.isHeldByCurrentThread(),
+ "Owner thread expected %s, got %s",
+ ownerLock.getOwner(),
+ Thread.currentThread());
+ }
+
+ /**
+ * Add a new writer's memory allocation to the pool. We use the path
+ * as a unique key to ensure that we don't get duplicates.
+ * @param path the file that is being written
+ * @param requestedAllocation the requested buffer size
+ */
+ public void addWriter(Path path, long requestedAllocation,
+ Callback callback) throws IOException {
+ checkOwner();
+ WriterInfo oldVal = writerList.get(path);
+ // this should always be null, but we handle the case where the memory
+ // manager wasn't told that a writer wasn't still in use and the task
+ // starts writing to the same path.
+ if (oldVal == null) {
+ oldVal = new WriterInfo(requestedAllocation, callback);
+ writerList.put(path, oldVal);
+ totalAllocation += requestedAllocation;
+ } else {
+ // handle a new writer that is writing to the same path
+ totalAllocation += requestedAllocation - oldVal.allocation;
+ oldVal.allocation = requestedAllocation;
+ oldVal.callback = callback;
+ }
+ updateScale(true);
+ }
+
+ /**
+ * Remove the given writer from the pool.
+ * @param path the file that has been closed
+ */
+ public void removeWriter(Path path) throws IOException {
+ checkOwner();
+ WriterInfo val = writerList.get(path);
+ if (val != null) {
+ writerList.remove(path);
+ totalAllocation -= val.allocation;
+ if (writerList.isEmpty()) {
+ rowsAddedSinceCheck = 0;
+ }
+ updateScale(false);
+ }
+ if(writerList.isEmpty()) {
+ rowsAddedSinceCheck = 0;
+ }
+ }
+
+ /**
+ * Get the total pool size that is available for ORC writers.
+ * @return the number of bytes in the pool
+ */
+ public long getTotalMemoryPool() {
+ return totalMemoryPool;
+ }
+
+ /**
+ * The scaling factor for each allocation to ensure that the pool isn't
+ * oversubscribed.
+ * @return a fraction between 0.0 and 1.0 of the requested size that is
+ * available for each writer.
+ */
+ public double getAllocationScale() {
+ return currentScale;
+ }
+
+ /**
+ * Give the memory manager an opportunity for doing a memory check.
+ * @param rows number of rows added
+ * @throws IOException
+ */
+ public void addedRow(int rows) throws IOException {
+ rowsAddedSinceCheck += rows;
+ if (rowsAddedSinceCheck >= ROWS_BETWEEN_CHECKS) {
+ notifyWriters();
+ }
+ }
+
+ /**
+ * Notify all of the writers that they should check their memory usage.
+ * @throws IOException
+ */
+ public void notifyWriters() throws IOException {
+ checkOwner();
+ LOG.debug("Notifying writers after " + rowsAddedSinceCheck);
+ for(WriterInfo writer: writerList.values()) {
+ boolean flushed = writer.callback.checkMemory(currentScale);
+ if (LOG.isDebugEnabled() && flushed) {
+ LOG.debug("flushed " + writer.toString());
+ }
+ }
+ rowsAddedSinceCheck = 0;
+ }
+
+ /**
+ * Update the currentScale based on the current allocation and pool size.
+ * This also updates the notificationTrigger.
+ * @param isAllocate is this an allocation?
+ */
+ private void updateScale(boolean isAllocate) throws IOException {
+ if (totalAllocation <= totalMemoryPool) {
+ currentScale = 1;
+ } else {
+ currentScale = (double) totalMemoryPool / totalAllocation;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/MetadataReader.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/MetadataReader.java b/orc/src/java/org/apache/orc/impl/MetadataReader.java
new file mode 100644
index 0000000..670a81a
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/MetadataReader.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+
+import org.apache.orc.OrcProto;
+import org.apache.orc.StripeInformation;
+
+public interface MetadataReader {
+ OrcIndex readRowIndex(StripeInformation stripe,
+ OrcProto.StripeFooter footer,
+ boolean[] included, OrcProto.RowIndex[] indexes, boolean[] sargColumns,
+ OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException;
+
+ OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException;
+
+ void close() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/MetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/MetadataReaderImpl.java b/orc/src/java/org/apache/orc/impl/MetadataReaderImpl.java
new file mode 100644
index 0000000..d0ded52
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/MetadataReaderImpl.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.orc.CompressionCodec;
+import org.apache.orc.OrcProto;
+import org.apache.orc.StripeInformation;
+
+import com.google.common.collect.Lists;
+
+public class MetadataReaderImpl implements MetadataReader {
+ private final FSDataInputStream file;
+ private final CompressionCodec codec;
+ private final int bufferSize;
+ private final int typeCount;
+
+ public MetadataReaderImpl(FileSystem fileSystem, Path path,
+ CompressionCodec codec, int bufferSize, int typeCount) throws IOException {
+ this(fileSystem.open(path), codec, bufferSize, typeCount);
+ }
+
+ public MetadataReaderImpl(FSDataInputStream file,
+ CompressionCodec codec, int bufferSize, int typeCount) {
+ this.file = file;
+ this.codec = codec;
+ this.bufferSize = bufferSize;
+ this.typeCount = typeCount;
+ }
+
+ @Override
+ public OrcIndex readRowIndex(StripeInformation stripe,
+ OrcProto.StripeFooter footer, boolean[] included, OrcProto.RowIndex[] indexes,
+ boolean[] sargColumns, OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException {
+ if (footer == null) {
+ footer = readStripeFooter(stripe);
+ }
+ if (indexes == null) {
+ indexes = new OrcProto.RowIndex[typeCount];
+ }
+ if (bloomFilterIndices == null) {
+ bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
+ }
+ long offset = stripe.getOffset();
+ List<OrcProto.Stream> streams = footer.getStreamsList();
+ for (int i = 0; i < streams.size(); i++) {
+ OrcProto.Stream stream = streams.get(i);
+ OrcProto.Stream nextStream = null;
+ if (i < streams.size() - 1) {
+ nextStream = streams.get(i+1);
+ }
+ int col = stream.getColumn();
+ int len = (int) stream.getLength();
+ // row index stream and bloom filter are interlaced, check if the sarg column contains bloom
+ // filter and combine the io to read row index and bloom filters for that column together
+ if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) {
+ boolean readBloomFilter = false;
+ if (sargColumns != null && sargColumns[col] &&
+ nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
+ len += nextStream.getLength();
+ i += 1;
+ readBloomFilter = true;
+ }
+ if ((included == null || included[col]) && indexes[col] == null) {
+ byte[] buffer = new byte[len];
+ file.readFully(offset, buffer, 0, buffer.length);
+ ByteBuffer bb = ByteBuffer.wrap(buffer);
+ indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
+ Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), stream.getLength(),
+ codec, bufferSize));
+ if (readBloomFilter) {
+ bb.position((int) stream.getLength());
+ bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(InStream.create(
+ "bloom_filter", Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)),
+ nextStream.getLength(), codec, bufferSize));
+ }
+ }
+ }
+ offset += len;
+ }
+
+ OrcIndex index = new OrcIndex(indexes, bloomFilterIndices);
+ return index;
+ }
+
+ @Override
+ public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
+ long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
+ int tailLength = (int) stripe.getFooterLength();
+
+ // read the footer
+ ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
+ file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
+ return OrcProto.StripeFooter.parseFrom(InStream.createCodedInputStream("footer",
+ Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
+ tailLength, codec, bufferSize));
+ }
+
+ @Override
+ public void close() throws IOException {
+ file.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/OrcIndex.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/OrcIndex.java b/orc/src/java/org/apache/orc/impl/OrcIndex.java
new file mode 100644
index 0000000..50a15f2
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/OrcIndex.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.orc.OrcProto;
+
+public final class OrcIndex {
+ OrcProto.RowIndex[] rowGroupIndex;
+ OrcProto.BloomFilterIndex[] bloomFilterIndex;
+
+ public OrcIndex(OrcProto.RowIndex[] rgIndex, OrcProto.BloomFilterIndex[] bfIndex) {
+ this.rowGroupIndex = rgIndex;
+ this.bloomFilterIndex = bfIndex;
+ }
+
+ public OrcProto.RowIndex[] getRowGroupIndex() {
+ return rowGroupIndex;
+ }
+
+ public OrcProto.BloomFilterIndex[] getBloomFilterIndex() {
+ return bloomFilterIndex;
+ }
+
+ public void setRowGroupIndex(OrcProto.RowIndex[] rowGroupIndex) {
+ this.rowGroupIndex = rowGroupIndex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/OutStream.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/OutStream.java b/orc/src/java/org/apache/orc/impl/OutStream.java
new file mode 100644
index 0000000..68ef7d4
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/OutStream.java
@@ -0,0 +1,289 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import org.apache.orc.CompressionCodec;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class OutStream extends PositionedOutputStream {
+
+ public interface OutputReceiver {
+ /**
+ * Output the given buffer to the final destination
+ * @param buffer the buffer to output
+ * @throws IOException
+ */
+ void output(ByteBuffer buffer) throws IOException;
+ }
+
+ public static final int HEADER_SIZE = 3;
+ private final String name;
+ private final OutputReceiver receiver;
+ // if enabled the stream will be suppressed when writing stripe
+ private boolean suppress;
+
+ /**
+ * Stores the uncompressed bytes that have been serialized, but not
+ * compressed yet. When this fills, we compress the entire buffer.
+ */
+ private ByteBuffer current = null;
+
+ /**
+ * Stores the compressed bytes until we have a full buffer and then outputs
+ * them to the receiver. If no compression is being done, this (and overflow)
+ * will always be null and the current buffer will be sent directly to the
+ * receiver.
+ */
+ private ByteBuffer compressed = null;
+
+ /**
+ * Since the compressed buffer may start with contents from previous
+ * compression blocks, we allocate an overflow buffer so that the
+ * output of the codec can be split between the two buffers. After the
+ * compressed buffer is sent to the receiver, the overflow buffer becomes
+ * the new compressed buffer.
+ */
+ private ByteBuffer overflow = null;
+ private final int bufferSize;
+ private final CompressionCodec codec;
+ private long compressedBytes = 0;
+ private long uncompressedBytes = 0;
+
+ public OutStream(String name,
+ int bufferSize,
+ CompressionCodec codec,
+ OutputReceiver receiver) throws IOException {
+ this.name = name;
+ this.bufferSize = bufferSize;
+ this.codec = codec;
+ this.receiver = receiver;
+ this.suppress = false;
+ }
+
+ public void clear() throws IOException {
+ flush();
+ suppress = false;
+ }
+
+ /**
+ * Write the length of the compressed bytes. Life is much easier if the
+ * header is constant length, so just use 3 bytes. Considering most of the
+ * codecs want between 32k (snappy) and 256k (lzo, zlib), 3 bytes should
+ * be plenty. We also use the low bit for whether it is the original or
+ * compressed bytes.
+ * @param buffer the buffer to write the header to
+ * @param position the position in the buffer to write at
+ * @param val the size in the file
+ * @param original is it uncompressed
+ */
+ private static void writeHeader(ByteBuffer buffer,
+ int position,
+ int val,
+ boolean original) {
+ buffer.put(position, (byte) ((val << 1) + (original ? 1 : 0)));
+ buffer.put(position + 1, (byte) (val >> 7));
+ buffer.put(position + 2, (byte) (val >> 15));
+ }
+
+ private void getNewInputBuffer() throws IOException {
+ if (codec == null) {
+ current = ByteBuffer.allocate(bufferSize);
+ } else {
+ current = ByteBuffer.allocate(bufferSize + HEADER_SIZE);
+ writeHeader(current, 0, bufferSize, true);
+ current.position(HEADER_SIZE);
+ }
+ }
+
+ /**
+ * Allocate a new output buffer if we are compressing.
+ */
+ private ByteBuffer getNewOutputBuffer() throws IOException {
+ return ByteBuffer.allocate(bufferSize + HEADER_SIZE);
+ }
+
+ private void flip() throws IOException {
+ current.limit(current.position());
+ current.position(codec == null ? 0 : HEADER_SIZE);
+ }
+
+ @Override
+ public void write(int i) throws IOException {
+ if (current == null) {
+ getNewInputBuffer();
+ }
+ if (current.remaining() < 1) {
+ spill();
+ }
+ uncompressedBytes += 1;
+ current.put((byte) i);
+ }
+
+ @Override
+ public void write(byte[] bytes, int offset, int length) throws IOException {
+ if (current == null) {
+ getNewInputBuffer();
+ }
+ int remaining = Math.min(current.remaining(), length);
+ current.put(bytes, offset, remaining);
+ uncompressedBytes += remaining;
+ length -= remaining;
+ while (length != 0) {
+ spill();
+ offset += remaining;
+ remaining = Math.min(current.remaining(), length);
+ current.put(bytes, offset, remaining);
+ uncompressedBytes += remaining;
+ length -= remaining;
+ }
+ }
+
+ private void spill() throws java.io.IOException {
+ // if there isn't anything in the current buffer, don't spill
+ if (current == null ||
+ current.position() == (codec == null ? 0 : HEADER_SIZE)) {
+ return;
+ }
+ flip();
+ if (codec == null) {
+ receiver.output(current);
+ getNewInputBuffer();
+ } else {
+ if (compressed == null) {
+ compressed = getNewOutputBuffer();
+ } else if (overflow == null) {
+ overflow = getNewOutputBuffer();
+ }
+ int sizePosn = compressed.position();
+ compressed.position(compressed.position() + HEADER_SIZE);
+ if (codec.compress(current, compressed, overflow)) {
+ uncompressedBytes = 0;
+ // move position back to after the header
+ current.position(HEADER_SIZE);
+ current.limit(current.capacity());
+ // find the total bytes in the chunk
+ int totalBytes = compressed.position() - sizePosn - HEADER_SIZE;
+ if (overflow != null) {
+ totalBytes += overflow.position();
+ }
+ compressedBytes += totalBytes + HEADER_SIZE;
+ writeHeader(compressed, sizePosn, totalBytes, false);
+ // if we have less than the next header left, spill it.
+ if (compressed.remaining() < HEADER_SIZE) {
+ compressed.flip();
+ receiver.output(compressed);
+ compressed = overflow;
+ overflow = null;
+ }
+ } else {
+ compressedBytes += uncompressedBytes + HEADER_SIZE;
+ uncompressedBytes = 0;
+ // we are using the original, but need to spill the current
+ // compressed buffer first. So back up to where we started,
+ // flip it and add it to done.
+ if (sizePosn != 0) {
+ compressed.position(sizePosn);
+ compressed.flip();
+ receiver.output(compressed);
+ compressed = null;
+ // if we have an overflow, clear it and make it the new compress
+ // buffer
+ if (overflow != null) {
+ overflow.clear();
+ compressed = overflow;
+ overflow = null;
+ }
+ } else {
+ compressed.clear();
+ if (overflow != null) {
+ overflow.clear();
+ }
+ }
+
+ // now add the current buffer into the done list and get a new one.
+ current.position(0);
+ // update the header with the current length
+ writeHeader(current, 0, current.limit() - HEADER_SIZE, true);
+ receiver.output(current);
+ getNewInputBuffer();
+ }
+ }
+ }
+
+ @Override
+ public void getPosition(PositionRecorder recorder) throws IOException {
+ if (codec == null) {
+ recorder.addPosition(uncompressedBytes);
+ } else {
+ recorder.addPosition(compressedBytes);
+ recorder.addPosition(uncompressedBytes);
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ spill();
+ if (compressed != null && compressed.position() != 0) {
+ compressed.flip();
+ receiver.output(compressed);
+ compressed = null;
+ }
+ uncompressedBytes = 0;
+ compressedBytes = 0;
+ overflow = null;
+ current = null;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @Override
+ public long getBufferSize() {
+ long result = 0;
+ if (current != null) {
+ result += current.capacity();
+ }
+ if (compressed != null) {
+ result += compressed.capacity();
+ }
+ if (overflow != null) {
+ result += overflow.capacity();
+ }
+ return result;
+ }
+
+ /**
+ * Set suppress flag
+ */
+ public void suppress() {
+ suppress = true;
+ }
+
+ /**
+ * Returns the state of suppress flag
+ * @return value of suppress flag
+ */
+ public boolean isSuppressed() {
+ return suppress;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/PositionProvider.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/PositionProvider.java b/orc/src/java/org/apache/orc/impl/PositionProvider.java
new file mode 100644
index 0000000..47cf481
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/PositionProvider.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+/**
+ * An interface used for seeking to a row index.
+ */
+public interface PositionProvider {
+ long getNext();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/PositionRecorder.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/PositionRecorder.java b/orc/src/java/org/apache/orc/impl/PositionRecorder.java
new file mode 100644
index 0000000..1fff760
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/PositionRecorder.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+/**
+ * An interface for recording positions in a stream.
+ */
+public interface PositionRecorder {
+ void addPosition(long offset);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/PositionedOutputStream.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/PositionedOutputStream.java b/orc/src/java/org/apache/orc/impl/PositionedOutputStream.java
new file mode 100644
index 0000000..d412939
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/PositionedOutputStream.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public abstract class PositionedOutputStream extends OutputStream {
+
+ /**
+ * Record the current position to the recorder.
+ * @param recorder the object that receives the position
+ * @throws IOException
+ */
+ public abstract void getPosition(PositionRecorder recorder
+ ) throws IOException;
+
+ /**
+ * Get the memory size currently allocated as buffer associated with this
+ * stream.
+ * @return the number of bytes used by buffers.
+ */
+ public abstract long getBufferSize();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/RedBlackTree.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/RedBlackTree.java b/orc/src/java/org/apache/orc/impl/RedBlackTree.java
new file mode 100644
index 0000000..41aa4b9
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/RedBlackTree.java
@@ -0,0 +1,311 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.orc.impl.DynamicIntArray;
+
+/**
+ * A memory efficient red-black tree that does not allocate any objects per
+ * an element. This class is abstract and assumes that the child class
+ * handles the key and comparisons with the key.
+ */
+abstract class RedBlackTree {
+ public static final int NULL = -1;
+
+ // Various values controlling the offset of the data within the array.
+ private static final int LEFT_OFFSET = 0;
+ private static final int RIGHT_OFFSET = 1;
+ private static final int ELEMENT_SIZE = 2;
+
+ protected int size = 0;
+ private final DynamicIntArray data;
+ protected int root = NULL;
+ protected int lastAdd = 0;
+ private boolean wasAdd = false;
+
+ /**
+ * Create a set with the given initial capacity.
+ */
+ public RedBlackTree(int initialCapacity) {
+ data = new DynamicIntArray(initialCapacity * ELEMENT_SIZE);
+ }
+
+ /**
+ * Insert a new node into the data array, growing the array as necessary.
+ *
+ * @return Returns the position of the new node.
+ */
+ private int insert(int left, int right, boolean isRed) {
+ int position = size;
+ size += 1;
+ setLeft(position, left, isRed);
+ setRight(position, right);
+ return position;
+ }
+
+ /**
+ * Compare the value at the given position to the new value.
+ * @return 0 if the values are the same, -1 if the new value is smaller and
+ * 1 if the new value is larger.
+ */
+ protected abstract int compareValue(int position);
+
+ /**
+ * Is the given node red as opposed to black? To prevent having an extra word
+ * in the data array, we just the low bit on the left child index.
+ */
+ protected boolean isRed(int position) {
+ return position != NULL &&
+ (data.get(position * ELEMENT_SIZE + LEFT_OFFSET) & 1) == 1;
+ }
+
+ /**
+ * Set the red bit true or false.
+ */
+ private void setRed(int position, boolean isRed) {
+ int offset = position * ELEMENT_SIZE + LEFT_OFFSET;
+ if (isRed) {
+ data.set(offset, data.get(offset) | 1);
+ } else {
+ data.set(offset, data.get(offset) & ~1);
+ }
+ }
+
+ /**
+ * Get the left field of the given position.
+ */
+ protected int getLeft(int position) {
+ return data.get(position * ELEMENT_SIZE + LEFT_OFFSET) >> 1;
+ }
+
+ /**
+ * Get the right field of the given position.
+ */
+ protected int getRight(int position) {
+ return data.get(position * ELEMENT_SIZE + RIGHT_OFFSET);
+ }
+
+ /**
+ * Set the left field of the given position.
+ * Note that we are storing the node color in the low bit of the left pointer.
+ */
+ private void setLeft(int position, int left) {
+ int offset = position * ELEMENT_SIZE + LEFT_OFFSET;
+ data.set(offset, (left << 1) | (data.get(offset) & 1));
+ }
+
+ /**
+ * Set the left field of the given position.
+ * Note that we are storing the node color in the low bit of the left pointer.
+ */
+ private void setLeft(int position, int left, boolean isRed) {
+ int offset = position * ELEMENT_SIZE + LEFT_OFFSET;
+ data.set(offset, (left << 1) | (isRed ? 1 : 0));
+ }
+
+ /**
+ * Set the right field of the given position.
+ */
+ private void setRight(int position, int right) {
+ data.set(position * ELEMENT_SIZE + RIGHT_OFFSET, right);
+ }
+
+ /**
+ * Insert or find a given key in the tree and rebalance the tree correctly.
+ * Rebalancing restores the red-black aspect of the tree to maintain the
+ * invariants:
+ * 1. If a node is red, both of its children are black.
+ * 2. Each child of a node has the same black height (the number of black
+ * nodes between it and the leaves of the tree).
+ *
+ * Inserted nodes are at the leaves and are red, therefore there is at most a
+ * violation of rule 1 at the node we just put in. Instead of always keeping
+ * the parents, this routine passing down the context.
+ *
+ * The fix is broken down into 6 cases (1.{1,2,3} and 2.{1,2,3} that are
+ * left-right mirror images of each other). See Algorighms by Cormen,
+ * Leiserson, and Rivest for the explaination of the subcases.
+ *
+ * @param node The node that we are fixing right now.
+ * @param fromLeft Did we come down from the left?
+ * @param parent Nodes' parent
+ * @param grandparent Parent's parent
+ * @param greatGrandparent Grandparent's parent
+ * @return Does parent also need to be checked and/or fixed?
+ */
+ private boolean add(int node, boolean fromLeft, int parent,
+ int grandparent, int greatGrandparent) {
+ if (node == NULL) {
+ if (root == NULL) {
+ lastAdd = insert(NULL, NULL, false);
+ root = lastAdd;
+ wasAdd = true;
+ return false;
+ } else {
+ lastAdd = insert(NULL, NULL, true);
+ node = lastAdd;
+ wasAdd = true;
+ // connect the new node into the tree
+ if (fromLeft) {
+ setLeft(parent, node);
+ } else {
+ setRight(parent, node);
+ }
+ }
+ } else {
+ int compare = compareValue(node);
+ boolean keepGoing;
+
+ // Recurse down to find where the node needs to be added
+ if (compare < 0) {
+ keepGoing = add(getLeft(node), true, node, parent, grandparent);
+ } else if (compare > 0) {
+ keepGoing = add(getRight(node), false, node, parent, grandparent);
+ } else {
+ lastAdd = node;
+ wasAdd = false;
+ return false;
+ }
+
+ // we don't need to fix the root (because it is always set to black)
+ if (node == root || !keepGoing) {
+ return false;
+ }
+ }
+
+
+ // Do we need to fix this node? Only if there are two reds right under each
+ // other.
+ if (isRed(node) && isRed(parent)) {
+ if (parent == getLeft(grandparent)) {
+ int uncle = getRight(grandparent);
+ if (isRed(uncle)) {
+ // case 1.1
+ setRed(parent, false);
+ setRed(uncle, false);
+ setRed(grandparent, true);
+ return true;
+ } else {
+ if (node == getRight(parent)) {
+ // case 1.2
+ // swap node and parent
+ int tmp = node;
+ node = parent;
+ parent = tmp;
+ // left-rotate on node
+ setLeft(grandparent, parent);
+ setRight(node, getLeft(parent));
+ setLeft(parent, node);
+ }
+
+ // case 1.2 and 1.3
+ setRed(parent, false);
+ setRed(grandparent, true);
+
+ // right-rotate on grandparent
+ if (greatGrandparent == NULL) {
+ root = parent;
+ } else if (getLeft(greatGrandparent) == grandparent) {
+ setLeft(greatGrandparent, parent);
+ } else {
+ setRight(greatGrandparent, parent);
+ }
+ setLeft(grandparent, getRight(parent));
+ setRight(parent, grandparent);
+ return false;
+ }
+ } else {
+ int uncle = getLeft(grandparent);
+ if (isRed(uncle)) {
+ // case 2.1
+ setRed(parent, false);
+ setRed(uncle, false);
+ setRed(grandparent, true);
+ return true;
+ } else {
+ if (node == getLeft(parent)) {
+ // case 2.2
+ // swap node and parent
+ int tmp = node;
+ node = parent;
+ parent = tmp;
+ // right-rotate on node
+ setRight(grandparent, parent);
+ setLeft(node, getRight(parent));
+ setRight(parent, node);
+ }
+ // case 2.2 and 2.3
+ setRed(parent, false);
+ setRed(grandparent, true);
+ // left-rotate on grandparent
+ if (greatGrandparent == NULL) {
+ root = parent;
+ } else if (getRight(greatGrandparent) == grandparent) {
+ setRight(greatGrandparent, parent);
+ } else {
+ setLeft(greatGrandparent, parent);
+ }
+ setRight(grandparent, getLeft(parent));
+ setLeft(parent, grandparent);
+ return false;
+ }
+ }
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * Add the new key to the tree.
+ * @return true if the element is a new one.
+ */
+ protected boolean add() {
+ add(root, false, NULL, NULL, NULL);
+ if (wasAdd) {
+ setRed(root, false);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Get the number of elements in the set.
+ */
+ public int size() {
+ return size;
+ }
+
+ /**
+ * Reset the table to empty.
+ */
+ public void clear() {
+ root = NULL;
+ size = 0;
+ data.clear();
+ }
+
+ /**
+ * Get the buffer size in bytes.
+ */
+ public long getSizeInBytes() {
+ return data.getSizeInBytes();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/RunLengthByteReader.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/RunLengthByteReader.java b/orc/src/java/org/apache/orc/impl/RunLengthByteReader.java
new file mode 100644
index 0000000..380f339
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/RunLengthByteReader.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+
+/**
+ * A reader that reads a sequence of bytes. A control byte is read before
+ * each run with positive values 0 to 127 meaning 3 to 130 repetitions. If the
+ * byte is -1 to -128, 1 to 128 literal byte values follow.
+ */
+public class RunLengthByteReader {
+ private InStream input;
+ private final byte[] literals =
+ new byte[RunLengthByteWriter.MAX_LITERAL_SIZE];
+ private int numLiterals = 0;
+ private int used = 0;
+ private boolean repeat = false;
+
+ public RunLengthByteReader(InStream input) throws IOException {
+ this.input = input;
+ }
+
+ public void setInStream(InStream input) {
+ this.input = input;
+ }
+
+ private void readValues(boolean ignoreEof) throws IOException {
+ int control = input.read();
+ used = 0;
+ if (control == -1) {
+ if (!ignoreEof) {
+ throw new EOFException("Read past end of buffer RLE byte from " + input);
+ }
+ used = numLiterals = 0;
+ return;
+ } else if (control < 0x80) {
+ repeat = true;
+ numLiterals = control + RunLengthByteWriter.MIN_REPEAT_SIZE;
+ int val = input.read();
+ if (val == -1) {
+ throw new EOFException("Reading RLE byte got EOF");
+ }
+ literals[0] = (byte) val;
+ } else {
+ repeat = false;
+ numLiterals = 0x100 - control;
+ int bytes = 0;
+ while (bytes < numLiterals) {
+ int result = input.read(literals, bytes, numLiterals - bytes);
+ if (result == -1) {
+ throw new EOFException("Reading RLE byte literal got EOF in " + this);
+ }
+ bytes += result;
+ }
+ }
+ }
+
+ public boolean hasNext() throws IOException {
+ return used != numLiterals || input.available() > 0;
+ }
+
+ public byte next() throws IOException {
+ byte result;
+ if (used == numLiterals) {
+ readValues(false);
+ }
+ if (repeat) {
+ result = literals[0];
+ } else {
+ result = literals[used];
+ }
+ ++used;
+ return result;
+ }
+
+ public void nextVector(LongColumnVector previous, long previousLen)
+ throws IOException {
+ previous.isRepeating = true;
+ for (int i = 0; i < previousLen; i++) {
+ if (!previous.isNull[i]) {
+ previous.vector[i] = next();
+ } else {
+ // The default value of null for int types in vectorized
+ // processing is 1, so set that if the value is null
+ previous.vector[i] = 1;
+ }
+
+ // The default value for nulls in Vectorization for int types is 1
+ // and given that non null value can also be 1, we need to check for isNull also
+ // when determining the isRepeating flag.
+ if (previous.isRepeating
+ && i > 0
+ && ((previous.vector[i - 1] != previous.vector[i]) || (previous.isNull[i - 1] != previous.isNull[i]))) {
+ previous.isRepeating = false;
+ }
+ }
+ }
+
+ public void seek(PositionProvider index) throws IOException {
+ input.seek(index);
+ int consumed = (int) index.getNext();
+ if (consumed != 0) {
+ // a loop is required for cases where we break the run into two parts
+ while (consumed > 0) {
+ readValues(false);
+ used = consumed;
+ consumed -= numLiterals;
+ }
+ } else {
+ used = 0;
+ numLiterals = 0;
+ }
+ }
+
+ public void skip(long items) throws IOException {
+ while (items > 0) {
+ if (used == numLiterals) {
+ readValues(false);
+ }
+ long consume = Math.min(items, numLiterals - used);
+ used += consume;
+ items -= consume;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "byte rle " + (repeat ? "repeat" : "literal") + " used: " +
+ used + "/" + numLiterals + " from " + input;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/RunLengthByteWriter.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/RunLengthByteWriter.java b/orc/src/java/org/apache/orc/impl/RunLengthByteWriter.java
new file mode 100644
index 0000000..09108b2
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/RunLengthByteWriter.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+
+/**
+ * A streamFactory that writes a sequence of bytes. A control byte is written before
+ * each run with positive values 0 to 127 meaning 2 to 129 repetitions. If the
+ * bytes is -1 to -128, 1 to 128 literal byte values follow.
+ */
+public class RunLengthByteWriter {
+ static final int MIN_REPEAT_SIZE = 3;
+ static final int MAX_LITERAL_SIZE = 128;
+ static final int MAX_REPEAT_SIZE= 127 + MIN_REPEAT_SIZE;
+ private final PositionedOutputStream output;
+ private final byte[] literals = new byte[MAX_LITERAL_SIZE];
+ private int numLiterals = 0;
+ private boolean repeat = false;
+ private int tailRunLength = 0;
+
+ public RunLengthByteWriter(PositionedOutputStream output) {
+ this.output = output;
+ }
+
+ private void writeValues() throws IOException {
+ if (numLiterals != 0) {
+ if (repeat) {
+ output.write(numLiterals - MIN_REPEAT_SIZE);
+ output.write(literals, 0, 1);
+ } else {
+ output.write(-numLiterals);
+ output.write(literals, 0, numLiterals);
+ }
+ repeat = false;
+ tailRunLength = 0;
+ numLiterals = 0;
+ }
+ }
+
+ public void flush() throws IOException {
+ writeValues();
+ output.flush();
+ }
+
+ public void write(byte value) throws IOException {
+ if (numLiterals == 0) {
+ literals[numLiterals++] = value;
+ tailRunLength = 1;
+ } else if (repeat) {
+ if (value == literals[0]) {
+ numLiterals += 1;
+ if (numLiterals == MAX_REPEAT_SIZE) {
+ writeValues();
+ }
+ } else {
+ writeValues();
+ literals[numLiterals++] = value;
+ tailRunLength = 1;
+ }
+ } else {
+ if (value == literals[numLiterals - 1]) {
+ tailRunLength += 1;
+ } else {
+ tailRunLength = 1;
+ }
+ if (tailRunLength == MIN_REPEAT_SIZE) {
+ if (numLiterals + 1 == MIN_REPEAT_SIZE) {
+ repeat = true;
+ numLiterals += 1;
+ } else {
+ numLiterals -= MIN_REPEAT_SIZE - 1;
+ writeValues();
+ literals[0] = value;
+ repeat = true;
+ numLiterals = MIN_REPEAT_SIZE;
+ }
+ } else {
+ literals[numLiterals++] = value;
+ if (numLiterals == MAX_LITERAL_SIZE) {
+ writeValues();
+ }
+ }
+ }
+ }
+
+ public void getPosition(PositionRecorder recorder) throws IOException {
+ output.getPosition(recorder);
+ recorder.addPosition(numLiterals);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/impl/RunLengthIntegerReader.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/impl/RunLengthIntegerReader.java b/orc/src/java/org/apache/orc/impl/RunLengthIntegerReader.java
new file mode 100644
index 0000000..f129c86
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/RunLengthIntegerReader.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+
+/**
+ * A reader that reads a sequence of integers.
+ * */
+public class RunLengthIntegerReader implements IntegerReader {
+ private InStream input;
+ private final boolean signed;
+ private final long[] literals =
+ new long[RunLengthIntegerWriter.MAX_LITERAL_SIZE];
+ private int numLiterals = 0;
+ private int delta = 0;
+ private int used = 0;
+ private boolean repeat = false;
+ private SerializationUtils utils;
+
+ public RunLengthIntegerReader(InStream input, boolean signed) throws IOException {
+ this.input = input;
+ this.signed = signed;
+ this.utils = new SerializationUtils();
+ }
+
+ private void readValues(boolean ignoreEof) throws IOException {
+ int control = input.read();
+ if (control == -1) {
+ if (!ignoreEof) {
+ throw new EOFException("Read past end of RLE integer from " + input);
+ }
+ used = numLiterals = 0;
+ return;
+ } else if (control < 0x80) {
+ numLiterals = control + RunLengthIntegerWriter.MIN_REPEAT_SIZE;
+ used = 0;
+ repeat = true;
+ delta = input.read();
+ if (delta == -1) {
+ throw new EOFException("End of stream in RLE Integer from " + input);
+ }
+ // convert from 0 to 255 to -128 to 127 by converting to a signed byte
+ delta = (byte) (0 + delta);
+ if (signed) {
+ literals[0] = utils.readVslong(input);
+ } else {
+ literals[0] = utils.readVulong(input);
+ }
+ } else {
+ repeat = false;
+ numLiterals = 0x100 - control;
+ used = 0;
+ for(int i=0; i < numLiterals; ++i) {
+ if (signed) {
+ literals[i] = utils.readVslong(input);
+ } else {
+ literals[i] = utils.readVulong(input);
+ }
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() throws IOException {
+ return used != numLiterals || input.available() > 0;
+ }
+
+ @Override
+ public long next() throws IOException {
+ long result;
+ if (used == numLiterals) {
+ readValues(false);
+ }
+ if (repeat) {
+ result = literals[0] + (used++) * delta;
+ } else {
+ result = literals[used++];
+ }
+ return result;
+ }
+
+ @Override
+ public void nextVector(LongColumnVector previous, long previousLen) throws IOException {
+ previous.isRepeating = true;
+ for (int i = 0; i < previousLen; i++) {
+ if (!previous.isNull[i]) {
+ previous.vector[i] = next();
+ } else {
+ // The default value of null for int type in vectorized
+ // processing is 1, so set that if the value is null
+ previous.vector[i] = 1;
+ }
+
+ // The default value for nulls in Vectorization for int types is 1
+ // and given that non null value can also be 1, we need to check for isNull also
+ // when determining the isRepeating flag.
+ if (previous.isRepeating
+ && i > 0
+ && (previous.vector[i - 1] != previous.vector[i] || previous.isNull[i - 1] != previous.isNull[i])) {
+ previous.isRepeating = false;
+ }
+ }
+ }
+
+ @Override
+ public void setInStream(InStream data) {
+ input = data;
+ }
+
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ input.seek(index);
+ int consumed = (int) index.getNext();
+ if (consumed != 0) {
+ // a loop is required for cases where we break the run into two parts
+ while (consumed > 0) {
+ readValues(false);
+ used = consumed;
+ consumed -= numLiterals;
+ }
+ } else {
+ used = 0;
+ numLiterals = 0;
+ }
+ }
+
+ @Override
+ public void skip(long numValues) throws IOException {
+ while (numValues > 0) {
+ if (used == numLiterals) {
+ readValues(false);
+ }
+ long consume = Math.min(numValues, numLiterals - used);
+ used += consume;
+ numValues -= consume;
+ }
+ }
+}
[06/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
deleted file mode 100644
index 227cfca..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
+++ /dev/null
@@ -1,497 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.ListIterator;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.common.io.DiskRange;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.CodedInputStream;
-
-public abstract class InStream extends InputStream {
-
- private static final Logger LOG = LoggerFactory.getLogger(InStream.class);
- private static final int PROTOBUF_MESSAGE_MAX_LIMIT = 1024 << 20; // 1GB
-
- protected final String name;
- protected long length;
-
- public InStream(String name, long length) {
- this.name = name;
- this.length = length;
- }
-
- public String getStreamName() {
- return name;
- }
-
- public long getStreamLength() {
- return length;
- }
-
- static class UncompressedStream extends InStream {
- private List<DiskRange> bytes;
- private long length;
- protected long currentOffset;
- private ByteBuffer range;
- private int currentRange;
-
- public UncompressedStream(String name, List<DiskRange> input, long length) {
- super(name, length);
- reset(input, length);
- }
-
- protected void reset(List<DiskRange> input, long length) {
- this.bytes = input;
- this.length = length;
- currentRange = 0;
- currentOffset = 0;
- range = null;
- }
-
- @Override
- public int read() {
- if (range == null || range.remaining() == 0) {
- if (currentOffset == length) {
- return -1;
- }
- seek(currentOffset);
- }
- currentOffset += 1;
- return 0xff & range.get();
- }
-
- @Override
- public int read(byte[] data, int offset, int length) {
- if (range == null || range.remaining() == 0) {
- if (currentOffset == this.length) {
- return -1;
- }
- seek(currentOffset);
- }
- int actualLength = Math.min(length, range.remaining());
- range.get(data, offset, actualLength);
- currentOffset += actualLength;
- return actualLength;
- }
-
- @Override
- public int available() {
- if (range != null && range.remaining() > 0) {
- return range.remaining();
- }
- return (int) (length - currentOffset);
- }
-
- @Override
- public void close() {
- currentRange = bytes.size();
- currentOffset = length;
- // explicit de-ref of bytes[]
- bytes.clear();
- }
-
- @Override
- public void seek(PositionProvider index) throws IOException {
- seek(index.getNext());
- }
-
- public void seek(long desired) {
- if (desired == 0 && bytes.isEmpty()) {
- logEmptySeek(name);
- return;
- }
- int i = 0;
- for (DiskRange curRange : bytes) {
- if (desired == 0 && curRange.getData().remaining() == 0) {
- logEmptySeek(name);
- return;
- }
- if (curRange.getOffset() <= desired &&
- (desired - curRange.getOffset()) < curRange.getLength()) {
- currentOffset = desired;
- currentRange = i;
- this.range = curRange.getData().duplicate();
- int pos = range.position();
- pos += (int)(desired - curRange.getOffset()); // this is why we duplicate
- this.range.position(pos);
- return;
- }
- ++i;
- }
- // if they are seeking to the precise end, go ahead and let them go there
- int segments = bytes.size();
- if (segments != 0 && desired == bytes.get(segments - 1).getEnd()) {
- currentOffset = desired;
- currentRange = segments - 1;
- DiskRange curRange = bytes.get(currentRange);
- this.range = curRange.getData().duplicate();
- int pos = range.position();
- pos += (int)(desired - curRange.getOffset()); // this is why we duplicate
- this.range.position(pos);
- return;
- }
- throw new IllegalArgumentException("Seek in " + name + " to " +
- desired + " is outside of the data");
- }
-
- @Override
- public String toString() {
- return "uncompressed stream " + name + " position: " + currentOffset +
- " length: " + length + " range: " + currentRange +
- " offset: " + (range == null ? 0 : range.position()) + " limit: " + (range == null ? 0 : range.limit());
- }
- }
-
- private static ByteBuffer allocateBuffer(int size, boolean isDirect) {
- // TODO: use the same pool as the ORC readers
- if (isDirect) {
- return ByteBuffer.allocateDirect(size);
- } else {
- return ByteBuffer.allocate(size);
- }
- }
-
- private static class CompressedStream extends InStream {
- private final List<DiskRange> bytes;
- private final int bufferSize;
- private ByteBuffer uncompressed;
- private final CompressionCodec codec;
- private ByteBuffer compressed;
- private long currentOffset;
- private int currentRange;
- private boolean isUncompressedOriginal;
-
- public CompressedStream(String name, List<DiskRange> input, long length,
- CompressionCodec codec, int bufferSize) {
- super(name, length);
- this.bytes = input;
- this.codec = codec;
- this.bufferSize = bufferSize;
- currentOffset = 0;
- currentRange = 0;
- }
-
- private void allocateForUncompressed(int size, boolean isDirect) {
- uncompressed = allocateBuffer(size, isDirect);
- }
-
- private void readHeader() throws IOException {
- if (compressed == null || compressed.remaining() <= 0) {
- seek(currentOffset);
- }
- if (compressed.remaining() > OutStream.HEADER_SIZE) {
- int b0 = compressed.get() & 0xff;
- int b1 = compressed.get() & 0xff;
- int b2 = compressed.get() & 0xff;
- boolean isOriginal = (b0 & 0x01) == 1;
- int chunkLength = (b2 << 15) | (b1 << 7) | (b0 >> 1);
-
- if (chunkLength > bufferSize) {
- throw new IllegalArgumentException("Buffer size too small. size = " +
- bufferSize + " needed = " + chunkLength);
- }
- // read 3 bytes, which should be equal to OutStream.HEADER_SIZE always
- assert OutStream.HEADER_SIZE == 3 : "The Orc HEADER_SIZE must be the same in OutStream and InStream";
- currentOffset += OutStream.HEADER_SIZE;
-
- ByteBuffer slice = this.slice(chunkLength);
-
- if (isOriginal) {
- uncompressed = slice;
- isUncompressedOriginal = true;
- } else {
- if (isUncompressedOriginal) {
- allocateForUncompressed(bufferSize, slice.isDirect());
- isUncompressedOriginal = false;
- } else if (uncompressed == null) {
- allocateForUncompressed(bufferSize, slice.isDirect());
- } else {
- uncompressed.clear();
- }
- codec.decompress(slice, uncompressed);
- }
- } else {
- throw new IllegalStateException("Can't read header at " + this);
- }
- }
-
- @Override
- public int read() throws IOException {
- if (uncompressed == null || uncompressed.remaining() == 0) {
- if (currentOffset == length) {
- return -1;
- }
- readHeader();
- }
- return 0xff & uncompressed.get();
- }
-
- @Override
- public int read(byte[] data, int offset, int length) throws IOException {
- if (uncompressed == null || uncompressed.remaining() == 0) {
- if (currentOffset == this.length) {
- return -1;
- }
- readHeader();
- }
- int actualLength = Math.min(length, uncompressed.remaining());
- uncompressed.get(data, offset, actualLength);
- return actualLength;
- }
-
- @Override
- public int available() throws IOException {
- if (uncompressed == null || uncompressed.remaining() == 0) {
- if (currentOffset == length) {
- return 0;
- }
- readHeader();
- }
- return uncompressed.remaining();
- }
-
- @Override
- public void close() {
- uncompressed = null;
- compressed = null;
- currentRange = bytes.size();
- currentOffset = length;
- bytes.clear();
- }
-
- @Override
- public void seek(PositionProvider index) throws IOException {
- seek(index.getNext());
- long uncompressedBytes = index.getNext();
- if (uncompressedBytes != 0) {
- readHeader();
- uncompressed.position(uncompressed.position() +
- (int) uncompressedBytes);
- } else if (uncompressed != null) {
- // mark the uncompressed buffer as done
- uncompressed.position(uncompressed.limit());
- }
- }
-
- /* slices a read only contiguous buffer of chunkLength */
- private ByteBuffer slice(int chunkLength) throws IOException {
- int len = chunkLength;
- final long oldOffset = currentOffset;
- ByteBuffer slice;
- if (compressed.remaining() >= len) {
- slice = compressed.slice();
- // simple case
- slice.limit(len);
- currentOffset += len;
- compressed.position(compressed.position() + len);
- return slice;
- } else if (currentRange >= (bytes.size() - 1)) {
- // nothing has been modified yet
- throw new IOException("EOF in " + this + " while trying to read " +
- chunkLength + " bytes");
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format(
- "Crossing into next BufferChunk because compressed only has %d bytes (needs %d)",
- compressed.remaining(), len));
- }
-
- // we need to consolidate 2 or more buffers into 1
- // first copy out compressed buffers
- ByteBuffer copy = allocateBuffer(chunkLength, compressed.isDirect());
- currentOffset += compressed.remaining();
- len -= compressed.remaining();
- copy.put(compressed);
- ListIterator<DiskRange> iter = bytes.listIterator(currentRange);
-
- while (len > 0 && iter.hasNext()) {
- ++currentRange;
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("Read slow-path, >1 cross block reads with %s", this.toString()));
- }
- DiskRange range = iter.next();
- compressed = range.getData().duplicate();
- if (compressed.remaining() >= len) {
- slice = compressed.slice();
- slice.limit(len);
- copy.put(slice);
- currentOffset += len;
- compressed.position(compressed.position() + len);
- return copy;
- }
- currentOffset += compressed.remaining();
- len -= compressed.remaining();
- copy.put(compressed);
- }
-
- // restore offsets for exception clarity
- seek(oldOffset);
- throw new IOException("EOF in " + this + " while trying to read " +
- chunkLength + " bytes");
- }
-
- private void seek(long desired) throws IOException {
- if (desired == 0 && bytes.isEmpty()) {
- logEmptySeek(name);
- return;
- }
- int i = 0;
- for (DiskRange range : bytes) {
- if (range.getOffset() <= desired && desired < range.getEnd()) {
- currentRange = i;
- compressed = range.getData().duplicate();
- int pos = compressed.position();
- pos += (int)(desired - range.getOffset());
- compressed.position(pos);
- currentOffset = desired;
- return;
- }
- ++i;
- }
- // if they are seeking to the precise end, go ahead and let them go there
- int segments = bytes.size();
- if (segments != 0 && desired == bytes.get(segments - 1).getEnd()) {
- DiskRange range = bytes.get(segments - 1);
- currentRange = segments - 1;
- compressed = range.getData().duplicate();
- compressed.position(compressed.limit());
- currentOffset = desired;
- return;
- }
- throw new IOException("Seek outside of data in " + this + " to " + desired);
- }
-
- private String rangeString() {
- StringBuilder builder = new StringBuilder();
- int i = 0;
- for (DiskRange range : bytes) {
- if (i != 0) {
- builder.append("; ");
- }
- builder.append(" range " + i + " = " + range.getOffset()
- + " to " + (range.getEnd() - range.getOffset()));
- ++i;
- }
- return builder.toString();
- }
-
- @Override
- public String toString() {
- return "compressed stream " + name + " position: " + currentOffset +
- " length: " + length + " range: " + currentRange +
- " offset: " + (compressed == null ? 0 : compressed.position()) + " limit: " + (compressed == null ? 0 : compressed.limit()) +
- rangeString() +
- (uncompressed == null ? "" :
- " uncompressed: " + uncompressed.position() + " to " +
- uncompressed.limit());
- }
- }
-
- public abstract void seek(PositionProvider index) throws IOException;
-
- private static void logEmptySeek(String name) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Attempting seek into empty stream (" + name + ") Skipping stream.");
- }
- }
-
- /**
- * Create an input stream from a list of buffers.
- * @param fileName name of the file
- * @param streamName the name of the stream
- * @param buffers the list of ranges of bytes for the stream
- * @param offsets a list of offsets (the same length as input) that must
- * contain the first offset of the each set of bytes in input
- * @param length the length in bytes of the stream
- * @param codec the compression codec
- * @param bufferSize the compression buffer size
- * @return an input stream
- * @throws IOException
- */
- @VisibleForTesting
- @Deprecated
- public static InStream create(String streamName,
- ByteBuffer[] buffers,
- long[] offsets,
- long length,
- CompressionCodec codec,
- int bufferSize) throws IOException {
- List<DiskRange> input = new ArrayList<DiskRange>(buffers.length);
- for (int i = 0; i < buffers.length; ++i) {
- input.add(new BufferChunk(buffers[i], offsets[i]));
- }
- return create(streamName, input, length, codec, bufferSize);
- }
-
- /**
- * Create an input stream from a list of disk ranges with data.
- * @param name the name of the stream
- * @param input the list of ranges of bytes for the stream; from disk or cache
- * @param length the length in bytes of the stream
- * @param codec the compression codec
- * @param bufferSize the compression buffer size
- * @param cache Low-level cache to use to put data, if any. Only works with compressed streams.
- * @return an input stream
- * @throws IOException
- */
- public static InStream create(String name,
- List<DiskRange> input,
- long length,
- CompressionCodec codec,
- int bufferSize) throws IOException {
- if (codec == null) {
- return new UncompressedStream(name, input, length);
- } else {
- return new CompressedStream(name, input, length, codec, bufferSize);
- }
- }
-
- /**
- * Creates coded input stream (used for protobuf message parsing) with higher message size limit.
- *
- * @param name the name of the stream
- * @param input the list of ranges of bytes for the stream; from disk or cache
- * @param length the length in bytes of the stream
- * @param codec the compression codec
- * @param bufferSize the compression buffer size
- * @return coded input stream
- * @throws IOException
- */
- public static CodedInputStream createCodedInputStream(
- String name,
- List<DiskRange> input,
- long length,
- CompressionCodec codec,
- int bufferSize) throws IOException {
- InStream inStream = create(name, input, length, codec, bufferSize);
- CodedInputStream codedInputStream = CodedInputStream.newInstance(inStream);
- codedInputStream.setSizeLimit(PROTOBUF_MESSAGE_MAX_LIMIT);
- return codedInputStream;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerColumnStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerColumnStatistics.java
deleted file mode 100644
index 15625fa..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerColumnStatistics.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-/**
- * Statistics for all of the integer columns, such as byte, short, int, and
- * long.
- */
-public interface IntegerColumnStatistics extends ColumnStatistics {
- /**
- * Get the smallest value in the column. Only defined if getNumberOfValues
- * is non-zero.
- * @return the minimum
- */
- long getMinimum();
-
- /**
- * Get the largest value in the column. Only defined if getNumberOfValues
- * is non-zero.
- * @return the maximum
- */
- long getMaximum();
-
- /**
- * Is the sum defined? If the sum overflowed the counter this will be false.
- * @return is the sum available
- */
- boolean isSumDefined();
-
- /**
- * Get the sum of the column. Only valid if isSumDefined returns true.
- * @return the sum of the column
- */
- long getSum();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerReader.java
deleted file mode 100644
index b1c7f0a..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerReader.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-
-/**
- * Interface for reading integers.
- */
-public interface IntegerReader {
-
- /**
- * Seek to the position provided by index.
- * @param index
- * @throws IOException
- */
- void seek(PositionProvider index) throws IOException;
-
- /**
- * Skip number of specified rows.
- * @param numValues
- * @throws IOException
- */
- void skip(long numValues) throws IOException;
-
- /**
- * Check if there are any more values left.
- * @return
- * @throws IOException
- */
- boolean hasNext() throws IOException;
-
- /**
- * Return the next available value.
- * @return
- * @throws IOException
- */
- long next() throws IOException;
-
- /**
- * Return the next available vector for values.
- * @return
- * @throws IOException
- */
- void nextVector(LongColumnVector previous, long previousLen)
- throws IOException;
-
- void setInStream(InStream data);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerWriter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerWriter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerWriter.java
deleted file mode 100644
index 775d02e..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerWriter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-
-/**
- * Interface for writing integers.
- */
-interface IntegerWriter {
-
- /**
- * Get position from the stream.
- * @param recorder
- * @throws IOException
- */
- void getPosition(PositionRecorder recorder) throws IOException;
-
- /**
- * Write the integer value
- * @param value
- * @throws IOException
- */
- void write(long value) throws IOException;
-
- /**
- * Flush the buffer
- * @throws IOException
- */
- void flush() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/JsonFileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/JsonFileDump.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/JsonFileDump.java
index f9a6d9e..b746390 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/JsonFileDump.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/JsonFileDump.java
@@ -26,6 +26,20 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.filters.BloomFilterIO;
import org.codehaus.jettison.json.JSONArray;
+import org.apache.orc.BinaryColumnStatistics;
+import org.apache.orc.BooleanColumnStatistics;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.impl.ColumnStatisticsImpl;
+import org.apache.orc.DateColumnStatistics;
+import org.apache.orc.DecimalColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.OrcProto;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
+import org.apache.orc.TimestampColumnStatistics;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.codehaus.jettison.json.JSONStringer;
@@ -151,7 +165,7 @@ public class JsonFileDump {
for (int colIdx : rowIndexCols) {
sargColumns[colIdx] = true;
}
- RecordReaderImpl.Index indices = rows.readRowIndex(stripeIx, null, sargColumns);
+ OrcIndex indices = rows.readRowIndex(stripeIx, null, sargColumns);
writer.key("indexes").array();
for (int col : rowIndexCols) {
writer.object();
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MemoryManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MemoryManager.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MemoryManager.java
deleted file mode 100644
index bb35b13..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MemoryManager.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import com.google.common.base.Preconditions;
-
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * Implements a memory manager that keeps a global context of how many ORC
- * writers there are and manages the memory between them. For use cases with
- * dynamic partitions, it is easy to end up with many writers in the same task.
- * By managing the size of each allocation, we try to cut down the size of each
- * allocation and keep the task from running out of memory.
- *
- * This class is not thread safe, but is re-entrant - ensure creation and all
- * invocations are triggered from the same thread.
- */
-class MemoryManager {
-
- private static final Logger LOG = LoggerFactory.getLogger(MemoryManager.class);
-
- /**
- * How often should we check the memory sizes? Measured in rows added
- * to all of the writers.
- */
- private static final int ROWS_BETWEEN_CHECKS = 5000;
- private final long totalMemoryPool;
- private final Map<Path, WriterInfo> writerList =
- new HashMap<Path, WriterInfo>();
- private long totalAllocation = 0;
- private double currentScale = 1;
- private int rowsAddedSinceCheck = 0;
- private final OwnedLock ownerLock = new OwnedLock();
-
- @SuppressWarnings("serial")
- private static class OwnedLock extends ReentrantLock {
- public Thread getOwner() {
- return super.getOwner();
- }
- }
-
- private static class WriterInfo {
- long allocation;
- Callback callback;
- WriterInfo(long allocation, Callback callback) {
- this.allocation = allocation;
- this.callback = callback;
- }
- }
-
- public interface Callback {
- /**
- * The writer needs to check its memory usage
- * @param newScale the current scale factor for memory allocations
- * @return true if the writer was over the limit
- * @throws IOException
- */
- boolean checkMemory(double newScale) throws IOException;
- }
-
- /**
- * Create the memory manager.
- * @param conf use the configuration to find the maximum size of the memory
- * pool.
- */
- MemoryManager(Configuration conf) {
- double maxLoad = OrcConf.MEMORY_POOL.getDouble(conf);
- totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().
- getHeapMemoryUsage().getMax() * maxLoad);
- ownerLock.lock();
- }
-
- /**
- * Light weight thread-safety check for multi-threaded access patterns
- */
- private void checkOwner() {
- Preconditions.checkArgument(ownerLock.isHeldByCurrentThread(),
- "Owner thread expected %s, got %s",
- ownerLock.getOwner(),
- Thread.currentThread());
- }
-
- /**
- * Add a new writer's memory allocation to the pool. We use the path
- * as a unique key to ensure that we don't get duplicates.
- * @param path the file that is being written
- * @param requestedAllocation the requested buffer size
- */
- void addWriter(Path path, long requestedAllocation,
- Callback callback) throws IOException {
- checkOwner();
- WriterInfo oldVal = writerList.get(path);
- // this should always be null, but we handle the case where the memory
- // manager wasn't told that a writer wasn't still in use and the task
- // starts writing to the same path.
- if (oldVal == null) {
- oldVal = new WriterInfo(requestedAllocation, callback);
- writerList.put(path, oldVal);
- totalAllocation += requestedAllocation;
- } else {
- // handle a new writer that is writing to the same path
- totalAllocation += requestedAllocation - oldVal.allocation;
- oldVal.allocation = requestedAllocation;
- oldVal.callback = callback;
- }
- updateScale(true);
- }
-
- /**
- * Remove the given writer from the pool.
- * @param path the file that has been closed
- */
- void removeWriter(Path path) throws IOException {
- checkOwner();
- WriterInfo val = writerList.get(path);
- if (val != null) {
- writerList.remove(path);
- totalAllocation -= val.allocation;
- if (writerList.isEmpty()) {
- rowsAddedSinceCheck = 0;
- }
- updateScale(false);
- }
- if(writerList.isEmpty()) {
- rowsAddedSinceCheck = 0;
- }
- }
-
- /**
- * Get the total pool size that is available for ORC writers.
- * @return the number of bytes in the pool
- */
- long getTotalMemoryPool() {
- return totalMemoryPool;
- }
-
- /**
- * The scaling factor for each allocation to ensure that the pool isn't
- * oversubscribed.
- * @return a fraction between 0.0 and 1.0 of the requested size that is
- * available for each writer.
- */
- double getAllocationScale() {
- return currentScale;
- }
-
- /**
- * Give the memory manager an opportunity for doing a memory check.
- * @param rows number of rows added
- * @throws IOException
- */
- void addedRow(int rows) throws IOException {
- rowsAddedSinceCheck += rows;
- if (rowsAddedSinceCheck >= ROWS_BETWEEN_CHECKS) {
- notifyWriters();
- }
- }
-
- /**
- * Notify all of the writers that they should check their memory usage.
- * @throws IOException
- */
- void notifyWriters() throws IOException {
- checkOwner();
- LOG.debug("Notifying writers after " + rowsAddedSinceCheck);
- for(WriterInfo writer: writerList.values()) {
- boolean flushed = writer.callback.checkMemory(currentScale);
- if (LOG.isDebugEnabled() && flushed) {
- LOG.debug("flushed " + writer.toString());
- }
- }
- rowsAddedSinceCheck = 0;
- }
-
- /**
- * Update the currentScale based on the current allocation and pool size.
- * This also updates the notificationTrigger.
- * @param isAllocate is this an allocation?
- */
- private void updateScale(boolean isAllocate) throws IOException {
- if (totalAllocation <= totalMemoryPool) {
- currentScale = 1;
- } else {
- currentScale = (double) totalMemoryPool / totalAllocation;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
deleted file mode 100644
index cea324c..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilterIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter;
-
-public interface MetadataReader {
- RecordReaderImpl.Index readRowIndex(StripeInformation stripe, StripeFooter footer,
- boolean[] included, RowIndex[] indexes, boolean[] sargColumns,
- BloomFilterIndex[] bloomFilterIndices) throws IOException;
-
- StripeFooter readStripeFooter(StripeInformation stripe) throws IOException;
-
- void close() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
deleted file mode 100644
index 4624927..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.io.DiskRange;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
-
-import com.google.common.collect.Lists;
-
-public class MetadataReaderImpl implements MetadataReader {
- private final FSDataInputStream file;
- private final CompressionCodec codec;
- private final int bufferSize;
- private final int typeCount;
-
- public MetadataReaderImpl(FileSystem fileSystem, Path path,
- CompressionCodec codec, int bufferSize, int typeCount) throws IOException {
- this(fileSystem.open(path), codec, bufferSize, typeCount);
- }
-
- public MetadataReaderImpl(FSDataInputStream file,
- CompressionCodec codec, int bufferSize, int typeCount) {
- this.file = file;
- this.codec = codec;
- this.bufferSize = bufferSize;
- this.typeCount = typeCount;
- }
-
- @Override
- public RecordReaderImpl.Index readRowIndex(StripeInformation stripe,
- OrcProto.StripeFooter footer, boolean[] included, OrcProto.RowIndex[] indexes,
- boolean[] sargColumns, OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException {
- if (footer == null) {
- footer = readStripeFooter(stripe);
- }
- if (indexes == null) {
- indexes = new OrcProto.RowIndex[typeCount];
- }
- if (bloomFilterIndices == null) {
- bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
- }
- long offset = stripe.getOffset();
- List<OrcProto.Stream> streams = footer.getStreamsList();
- for (int i = 0; i < streams.size(); i++) {
- OrcProto.Stream stream = streams.get(i);
- OrcProto.Stream nextStream = null;
- if (i < streams.size() - 1) {
- nextStream = streams.get(i+1);
- }
- int col = stream.getColumn();
- int len = (int) stream.getLength();
- // row index stream and bloom filter are interlaced, check if the sarg column contains bloom
- // filter and combine the io to read row index and bloom filters for that column together
- if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) {
- boolean readBloomFilter = false;
- if (sargColumns != null && sargColumns[col] &&
- nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
- len += nextStream.getLength();
- i += 1;
- readBloomFilter = true;
- }
- if ((included == null || included[col]) && indexes[col] == null) {
- byte[] buffer = new byte[len];
- file.readFully(offset, buffer, 0, buffer.length);
- ByteBuffer bb = ByteBuffer.wrap(buffer);
- indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
- Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), stream.getLength(),
- codec, bufferSize));
- if (readBloomFilter) {
- bb.position((int) stream.getLength());
- bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(InStream.create(
- "bloom_filter", Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)),
- nextStream.getLength(), codec, bufferSize));
- }
- }
- }
- offset += len;
- }
-
- RecordReaderImpl.Index index = new RecordReaderImpl.Index(indexes, bloomFilterIndices);
- return index;
- }
-
- @Override
- public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
- long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
- int tailLength = (int) stripe.getFooterLength();
-
- // read the footer
- ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
- file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
- return OrcProto.StripeFooter.parseFrom(InStream.createCodedInputStream("footer",
- Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
- tailLength, codec, bufferSize));
- }
-
- @Override
- public void close() throws IOException {
- file.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java
deleted file mode 100644
index 132889c..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcConf.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import org.apache.hadoop.conf.Configuration;
-
-import java.util.Properties;
-
-/**
- * Define the configuration properties that Orc understands.
- */
-public enum OrcConf {
- STRIPE_SIZE("orc.stripe.size", "hive.exec.orc.default.stripe.size",
- 64L * 1024 * 1024,
- "Define the default ORC stripe size, in bytes."),
- BLOCK_SIZE("orc.block.size", "hive.exec.orc.default.block.size",
- 256L * 1024 * 1024,
- "Define the default file system block size for ORC files."),
- ENABLE_INDEXES("orc.create.index", "orc.create.index", true,
- "Should the ORC writer create indexes as part of the file."),
- ROW_INDEX_STRIDE("orc.row.index.stride",
- "hive.exec.orc.default.row.index.stride", 10000,
- "Define the default ORC index stride in number of rows. (Stride is the\n"+
- " number of rows n index entry represents.)"),
- BUFFER_SIZE("orc.compress.size", "hive.exec.orc.default.buffer.size",
- 256 * 1024, "Define the default ORC buffer size, in bytes."),
- BLOCK_PADDING("orc.block.padding", "hive.exec.orc.default.block.padding",
- true,
- "Define whether stripes should be padded to the HDFS block boundaries."),
- COMPRESS("orc.compress", "hive.exec.orc.default.compress", "ZLIB",
- "Define the default compression codec for ORC file"),
- WRITE_FORMAT("orc.write.format", "hive.exec.orc.write.format", "0.12",
- "Define the version of the file to write. Possible values are 0.11 and\n"+
- " 0.12. If this parameter is not defined, ORC will use the run\n" +
- " length encoding (RLE) introduced in Hive 0.12."),
- ENCODING_STRATEGY("orc.encoding.strategy", "hive.exec.orc.encoding.strategy",
- "SPEED",
- "Define the encoding strategy to use while writing data. Changing this\n"+
- "will only affect the light weight encoding for integers. This\n" +
- "flag will not change the compression level of higher level\n" +
- "compression codec (like ZLIB)."),
- COMPRESSION_STRATEGY("orc.compression.strategy",
- "hive.exec.orc.compression.strategy", "SPEED",
- "Define the compression strategy to use while writing data.\n" +
- "This changes the compression level of higher level compression\n" +
- "codec (like ZLIB)."),
- BLOCK_PADDING_TOLERANCE("orc.block.padding.tolerance",
- "hive.exec.orc.block.padding.tolerance", 0.05,
- "Define the tolerance for block padding as a decimal fraction of\n" +
- "stripe size (for example, the default value 0.05 is 5% of the\n" +
- "stripe size). For the defaults of 64Mb ORC stripe and 256Mb HDFS\n" +
- "blocks, the default block padding tolerance of 5% will\n" +
- "reserve a maximum of 3.2Mb for padding within the 256Mb block.\n" +
- "In that case, if the available size within the block is more than\n"+
- "3.2Mb, a new smaller stripe will be inserted to fit within that\n" +
- "space. This will make sure that no stripe written will block\n" +
- " boundaries and cause remote reads within a node local task."),
- BLOOM_FILTER_FPP("orc.bloom.filter.fpp", "orc.default.bloom.fpp", 0.05,
- "Define the default false positive probability for bloom filters."),
- USE_ZEROCOPY("orc.use.zerocopy", "hive.exec.orc.zerocopy", false,
- "Use zerocopy reads with ORC. (This requires Hadoop 2.3 or later.)"),
- SKIP_CORRUPT_DATA("orc.skip.corrupt.data", "hive.exec.orc.skip.corrupt.data",
- false,
- "If ORC reader encounters corrupt data, this value will be used to\n" +
- "determine whether to skip the corrupt data or throw exception.\n" +
- "The default behavior is to throw exception."),
- MEMORY_POOL("orc.memory.pool", "hive.exec.orc.memory.pool", 0.5,
- "Maximum fraction of heap that can be used by ORC file writers"),
- DICTIONARY_KEY_SIZE_THRESHOLD("orc.dictionary.key.threshold",
- "hive.exec.orc.dictionary.key.size.threshold",
- 0.8,
- "If the number of distinct keys in a dictionary is greater than this\n" +
- "fraction of the total number of non-null rows, turn off \n" +
- "dictionary encoding. Use 1 to always use dictionary encoding."),
- ROW_INDEX_STRIDE_DICTIONARY_CHECK("orc.dictionary.early.check",
- "hive.orc.row.index.stride.dictionary.check",
- true,
- "If enabled dictionary check will happen after first row index stride\n" +
- "(default 10000 rows) else dictionary check will happen before\n" +
- "writing first stripe. In both cases, the decision to use\n" +
- "dictionary or not will be retained thereafter."),
- BLOOM_FILTER_COLUMNS("orc.bloom.filter.columns", "orc.bloom.filter.columns",
- "", "List of columns to create bloom filters for when writing.")
- ;
-
- private final String attribute;
- private final String hiveConfName;
- private final Object defaultValue;
- private final String description;
-
- OrcConf(String attribute,
- String hiveConfName,
- Object defaultValue,
- String description) {
- this.attribute = attribute;
- this.hiveConfName = hiveConfName;
- this.defaultValue = defaultValue;
- this.description = description;
- }
-
- public String getAttribute() {
- return attribute;
- }
-
- public String getHiveConfName() {
- return hiveConfName;
- }
-
- public Object getDefaultValue() {
- return defaultValue;
- }
-
- public String getDescription() {
- return description;
- }
-
- private String lookupValue(Properties tbl, Configuration conf) {
- String result = null;
- if (tbl != null) {
- result = tbl.getProperty(attribute);
- }
- if (result == null && conf != null) {
- result = conf.get(attribute);
- if (result == null) {
- result = conf.get(hiveConfName);
- }
- }
- return result;
- }
-
- public long getLong(Properties tbl, Configuration conf) {
- String value = lookupValue(tbl, conf);
- if (value != null) {
- return Long.parseLong(value);
- }
- return ((Number) defaultValue).longValue();
- }
-
- public long getLong(Configuration conf) {
- return getLong(null, conf);
- }
-
- public String getString(Properties tbl, Configuration conf) {
- String value = lookupValue(tbl, conf);
- return value == null ? (String) defaultValue : value;
- }
-
- public String getString(Configuration conf) {
- return getString(null, conf);
- }
-
- public boolean getBoolean(Properties tbl, Configuration conf) {
- String value = lookupValue(tbl, conf);
- if (value != null) {
- return Boolean.parseBoolean(value);
- }
- return (Boolean) defaultValue;
- }
-
- public boolean getBoolean(Configuration conf) {
- return getBoolean(null, conf);
- }
-
- public double getDouble(Properties tbl, Configuration conf) {
- String value = lookupValue(tbl, conf);
- if (value != null) {
- return Double.parseDouble(value);
- }
- return ((Number) defaultValue).doubleValue();
- }
-
- public double getDouble(Configuration conf) {
- return getDouble(null, conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
index 56ac40b..975825a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.hive.ql.io.orc;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
@@ -28,132 +26,17 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.orc.FileMetadata;
+import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.TypeDescription;
/**
* Contains factory methods to read or write ORC files.
*/
-public final class OrcFile {
-
- public static final String MAGIC = "ORC";
-
- /**
- * Create a version number for the ORC file format, so that we can add
- * non-forward compatible changes in the future. To make it easier for users
- * to understand the version numbers, we use the Hive release number that
- * first wrote that version of ORC files.
- *
- * Thus, if you add new encodings or other non-forward compatible changes
- * to ORC files, which prevent the old reader from reading the new format,
- * you should change these variable to reflect the next Hive release number.
- * Non-forward compatible changes should never be added in patch releases.
- *
- * Do not make any changes that break backwards compatibility, which would
- * prevent the new reader from reading ORC files generated by any released
- * version of Hive.
- */
- public enum Version {
- V_0_11("0.11", 0, 11),
- V_0_12("0.12", 0, 12);
-
- public static final Version CURRENT = V_0_12;
-
- private final String name;
- private final int major;
- private final int minor;
-
- Version(String name, int major, int minor) {
- this.name = name;
- this.major = major;
- this.minor = minor;
- }
-
- public static Version byName(String name) {
- for(Version version: values()) {
- if (version.name.equals(name)) {
- return version;
- }
- }
- throw new IllegalArgumentException("Unknown ORC version " + name);
- }
-
- /**
- * Get the human readable name for the version.
- */
- public String getName() {
- return name;
- }
-
- /**
- * Get the major version number.
- */
- public int getMajor() {
- return major;
- }
-
- /**
- * Get the minor version number.
- */
- public int getMinor() {
- return minor;
- }
- }
-
- /**
- * Records the version of the writer in terms of which bugs have been fixed.
- * For bugs in the writer, but the old readers already read the new data
- * correctly, bump this version instead of the Version.
- */
- public enum WriterVersion {
- ORIGINAL(0),
- HIVE_8732(1), // corrupted stripe/file maximum column statistics
- HIVE_4243(2), // use real column names from Hive tables
-// Don't use any magic numbers here except for the below:
- FUTURE(Integer.MAX_VALUE); // a version from a future writer
-
- private final int id;
-
- public int getId() {
- return id;
- }
-
- WriterVersion(int id) {
- this.id = id;
- }
-
- private static final WriterVersion[] values;
- static {
- // Assumes few non-negative values close to zero.
- int max = Integer.MIN_VALUE;
- for (WriterVersion v : WriterVersion.values()) {
- if (v.id < 0) throw new AssertionError();
- if (v.id > max && FUTURE.id != v.id) {
- max = v.id;
- }
- }
- values = new WriterVersion[max + 1];
- for (WriterVersion v : WriterVersion.values()) {
- if (v.id < values.length) {
- values[v.id] = v;
- }
- }
- }
-
- public static WriterVersion from(int val) {
- if (val == FUTURE.id) return FUTURE; // Special handling for the magic value.
- return values[val];
- }
- }
-
- public enum EncodingStrategy {
- SPEED, COMPRESSION
- }
-
- public enum CompressionStrategy {
- SPEED, COMPRESSION
- }
+public final class OrcFile extends org.apache.orc.OrcFile {
// unused
- private OrcFile() {}
+ protected OrcFile() {}
/**
* Create an ORC file reader.
@@ -162,62 +45,32 @@ public final class OrcFile {
* @return a new ORC file reader.
* @throws IOException
*/
- public static Reader createReader(FileSystem fs, Path path
- ) throws IOException {
+ public static Reader createReader(FileSystem fs,
+ Path path) throws IOException {
ReaderOptions opts = new ReaderOptions(new Configuration());
opts.filesystem(fs);
return new ReaderImpl(path, opts);
}
- public static class ReaderOptions {
- private final Configuration conf;
- private FileSystem filesystem;
- private FileMetaInfo fileMetaInfo; // TODO: this comes from some place.
- private long maxLength = Long.MAX_VALUE;
- private FileMetadata fullFileMetadata; // Propagate from LLAP cache.
-
+ public static class ReaderOptions extends org.apache.orc.OrcFile.ReaderOptions {
public ReaderOptions(Configuration conf) {
- this.conf = conf;
- }
- ReaderOptions fileMetaInfo(FileMetaInfo info) {
- fileMetaInfo = info;
- return this;
+ super(conf);
}
public ReaderOptions filesystem(FileSystem fs) {
- this.filesystem = fs;
+ super.filesystem(fs);
return this;
}
public ReaderOptions maxLength(long val) {
- maxLength = val;
+ super.maxLength(val);
return this;
}
public ReaderOptions fileMetadata(FileMetadata metadata) {
- this.fullFileMetadata = metadata;
+ super.fileMetadata(metadata);
return this;
}
-
- Configuration getConfiguration() {
- return conf;
- }
-
- FileSystem getFilesystem() {
- return filesystem;
- }
-
- FileMetaInfo getFileMetaInfo() {
- return fileMetaInfo;
- }
-
- long getMaxLength() {
- return maxLength;
- }
-
- FileMetadata getFileMetadata() {
- return fullFileMetadata;
- }
}
public static ReaderOptions readerOptions(Configuration conf) {
@@ -229,71 +82,39 @@ public final class OrcFile {
return new ReaderImpl(path, options);
}
- public interface WriterContext {
- Writer getWriter();
- }
-
- public interface WriterCallback {
- void preStripeWrite(WriterContext context) throws IOException;
- void preFooterWrite(WriterContext context) throws IOException;
- }
-
/**
* Options for creating ORC file writers.
*/
- public static class WriterOptions {
- private final Configuration configuration;
- private FileSystem fileSystemValue = null;
+ public static class WriterOptions extends org.apache.orc.OrcFile.WriterOptions {
private boolean explicitSchema = false;
- private TypeDescription schema = null;
private ObjectInspector inspector = null;
- private long stripeSizeValue;
- private long blockSizeValue;
- private int rowIndexStrideValue;
- private int bufferSizeValue;
- private boolean blockPaddingValue;
- private CompressionKind compressValue;
- private MemoryManager memoryManagerValue;
- private Version versionValue;
- private WriterCallback callback;
- private EncodingStrategy encodingStrategy;
- private CompressionStrategy compressionStrategy;
- private double paddingTolerance;
- private String bloomFilterColumns;
- private double bloomFilterFpp;
WriterOptions(Properties tableProperties, Configuration conf) {
- configuration = conf;
- memoryManagerValue = getMemoryManager(conf);
- stripeSizeValue = OrcConf.STRIPE_SIZE.getLong(tableProperties, conf);
- blockSizeValue = OrcConf.BLOCK_SIZE.getLong(tableProperties, conf);
- rowIndexStrideValue =
- (int) OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf);
- bufferSizeValue = (int) OrcConf.BUFFER_SIZE.getLong(tableProperties,
- conf);
- blockPaddingValue =
- OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf);
- compressValue =
- CompressionKind.valueOf(OrcConf.COMPRESS.getString(tableProperties,
- conf));
- String versionName = OrcConf.WRITE_FORMAT.getString(tableProperties,
- conf);
- versionValue = Version.byName(versionName);
- String enString = OrcConf.ENCODING_STRATEGY.getString(tableProperties,
- conf);
- encodingStrategy = EncodingStrategy.valueOf(enString);
-
- String compString =
- OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf);
- compressionStrategy = CompressionStrategy.valueOf(compString);
-
- paddingTolerance =
- OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf);
-
- bloomFilterColumns = OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties,
- conf);
- bloomFilterFpp = OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties,
- conf);
+ super(tableProperties, conf);
+ }
+
+ /**
+ * A required option that sets the object inspector for the rows. If
+ * setSchema is not called, it also defines the schema.
+ */
+ public WriterOptions inspector(ObjectInspector value) {
+ this.inspector = value;
+ if (!explicitSchema) {
+ super.setSchema(OrcInputFormat.convertTypeInfo(
+ TypeInfoUtils.getTypeInfoFromObjectInspector(value)));
+ }
+ return this;
+ }
+
+ /**
+ * Set the schema for the file. This is a required parameter.
+ * @param schema the schema for the file.
+ * @return this
+ */
+ public WriterOptions setSchema(TypeDescription schema) {
+ this.explicitSchema = true;
+ super.setSchema(schema);
+ return this;
}
/**
@@ -301,7 +122,7 @@ public final class OrcFile {
* If it is not provided, it will be found from the path.
*/
public WriterOptions fileSystem(FileSystem value) {
- fileSystemValue = value;
+ super.fileSystem(value);
return this;
}
@@ -311,7 +132,7 @@ public final class OrcFile {
* is flushed to the HDFS file and the next stripe started.
*/
public WriterOptions stripeSize(long value) {
- stripeSizeValue = value;
+ super.stripeSize(value);
return this;
}
@@ -320,7 +141,7 @@ public final class OrcFile {
* set the block size to be multiple factors of stripe size.
*/
public WriterOptions blockSize(long value) {
- blockSizeValue = value;
+ super.blockSize(value);
return this;
}
@@ -330,7 +151,7 @@ public final class OrcFile {
* set to 0, no indexes will be included in the file.
*/
public WriterOptions rowIndexStride(int value) {
- rowIndexStrideValue = value;
+ super.rowIndexStride(value);
return this;
}
@@ -339,7 +160,7 @@ public final class OrcFile {
* stripe in memory.
*/
public WriterOptions bufferSize(int value) {
- bufferSizeValue = value;
+ super.bufferSize(value);
return this;
}
@@ -349,7 +170,7 @@ public final class OrcFile {
* reading, but costs space.
*/
public WriterOptions blockPadding(boolean value) {
- blockPaddingValue = value;
+ super.blockPadding(value);
return this;
}
@@ -357,7 +178,7 @@ public final class OrcFile {
* Sets the encoding strategy that is used to encode the data.
*/
public WriterOptions encodingStrategy(EncodingStrategy strategy) {
- encodingStrategy = strategy;
+ super.encodingStrategy(strategy);
return this;
}
@@ -365,7 +186,7 @@ public final class OrcFile {
* Sets the tolerance for block padding as a percentage of stripe size.
*/
public WriterOptions paddingTolerance(double value) {
- paddingTolerance = value;
+ super.paddingTolerance(value);
return this;
}
@@ -373,7 +194,7 @@ public final class OrcFile {
* Comma separated values of column names for which bloom filter is to be created.
*/
public WriterOptions bloomFilterColumns(String columns) {
- bloomFilterColumns = columns;
+ super.bloomFilterColumns(columns);
return this;
}
@@ -383,7 +204,7 @@ public final class OrcFile {
* @return this
*/
public WriterOptions bloomFilterFpp(double fpp) {
- bloomFilterFpp = fpp;
+ super.bloomFilterFpp(fpp);
return this;
}
@@ -391,31 +212,15 @@ public final class OrcFile {
* Sets the generic compression that is used to compress the data.
*/
public WriterOptions compress(CompressionKind value) {
- compressValue = value;
+ super.compress(value.getUnderlying());
return this;
}
/**
- * A required option that sets the object inspector for the rows. If
- * setSchema is not called, it also defines the schema.
- */
- public WriterOptions inspector(ObjectInspector value) {
- this.inspector = value;
- if (!explicitSchema) {
- schema = OrcUtils.convertTypeInfo(
- TypeInfoUtils.getTypeInfoFromObjectInspector(value));
- }
- return this;
- }
-
- /**
- * Set the schema for the file. This is a required parameter.
- * @param schema the schema for the file.
- * @return this
+ * Sets the generic compression that is used to compress the data.
*/
- public WriterOptions setSchema(TypeDescription schema) {
- this.explicitSchema = true;
- this.schema = schema;
+ public WriterOptions compress(org.apache.orc.CompressionKind value) {
+ super.compress(value);
return this;
}
@@ -423,7 +228,7 @@ public final class OrcFile {
* Sets the version of the file that will be written.
*/
public WriterOptions version(Version value) {
- versionValue = value;
+ super.version(value);
return this;
}
@@ -433,18 +238,17 @@ public final class OrcFile {
* @return this
*/
public WriterOptions callback(WriterCallback callback) {
- this.callback = callback;
+ super.callback(callback);
return this;
}
/**
* A package local option to set the memory manager.
*/
- WriterOptions memory(MemoryManager value) {
- memoryManagerValue = value;
+ protected WriterOptions memory(MemoryManager value) {
+ super.memory(value);
return this;
}
-
}
/**
@@ -479,18 +283,19 @@ public final class OrcFile {
public static Writer createWriter(Path path,
WriterOptions opts
) throws IOException {
- FileSystem fs = opts.fileSystemValue == null ?
- path.getFileSystem(opts.configuration) : opts.fileSystemValue;
-
- return new WriterImpl(fs, path, opts.configuration, opts.inspector,
- opts.schema,
- opts.stripeSizeValue, opts.compressValue,
- opts.bufferSizeValue, opts.rowIndexStrideValue,
- opts.memoryManagerValue, opts.blockPaddingValue,
- opts.versionValue, opts.callback,
- opts.encodingStrategy, opts.compressionStrategy,
- opts.paddingTolerance, opts.blockSizeValue,
- opts.bloomFilterColumns, opts.bloomFilterFpp);
+ FileSystem fs = opts.getFileSystem() == null ?
+ path.getFileSystem(opts.getConfiguration()) : opts.getFileSystem();
+
+ return new WriterImpl(fs, path, opts.getConfiguration(), opts.inspector,
+ opts.getSchema(),
+ opts.getStripeSize(), opts.getCompress(),
+ opts.getBufferSize(), opts.getRowIndexStride(),
+ opts.getMemoryManager(), opts.getBlockPadding(),
+ opts.getVersion(), opts.getCallback(),
+ opts.getEncodingStrategy(),
+ opts.getCompressionStrategy(),
+ opts.getPaddingTolerance(), opts.getBlockSize(),
+ opts.getBloomFilterColumns(), opts.getBloomFilterFpp());
}
/**
@@ -515,29 +320,12 @@ public final class OrcFile {
CompressionKind compress,
int bufferSize,
int rowIndexStride) throws IOException {
- return createWriter(path,
- writerOptions(conf)
- .fileSystem(fs)
- .inspector(inspector)
- .stripeSize(stripeSize)
- .compress(compress)
- .bufferSize(bufferSize)
- .rowIndexStride(rowIndexStride));
- }
-
- private static ThreadLocal<MemoryManager> memoryManager = null;
-
- private static synchronized MemoryManager getMemoryManager(
- final Configuration conf) {
- if (memoryManager == null) {
- memoryManager = new ThreadLocal<MemoryManager>() {
- @Override
- protected MemoryManager initialValue() {
- return new MemoryManager(conf);
- }
- };
- }
- return memoryManager.get();
+ return createWriter(path, writerOptions(conf)
+ .inspector(inspector)
+ .fileSystem(fs)
+ .stripeSize(stripeSize)
+ .compress(compress)
+ .bufferSize(bufferSize)
+ .rowIndexStride(rowIndexStride));
}
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
index 8eda37f..40f1da0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
@@ -25,6 +25,8 @@ import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.WritableComparable;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcProto;
/**
* Key for OrcFileMergeMapper task. Contains orc file related information that
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
index 41a97a3..f06195f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.RecordReader;
+import org.apache.orc.OrcProto;
+import org.apache.orc.StripeInformation;
public class OrcFileStripeMergeRecordReader implements
RecordReader<OrcFileKeyWrapper, OrcFileValueWrapper> {
@@ -98,7 +100,7 @@ public class OrcFileStripeMergeRecordReader implements
valueWrapper.setUserMetadata(((ReaderImpl) reader).getOrcProtoUserMetadata());
}
keyWrapper.setInputPath(path);
- keyWrapper.setCompression(reader.getCompression());
+ keyWrapper.setCompression(reader.getCompressionKind());
keyWrapper.setCompressBufferSize(reader.getCompressionSize());
keyWrapper.setVersion(reader.getFileVersion());
keyWrapper.setRowIndexStride(reader.getRowIndexStride());
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileValueWrapper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileValueWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileValueWrapper.java
index 7389ef5..846c874 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileValueWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileValueWrapper.java
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.io.WritableComparable;
+import org.apache.orc.OrcProto;
+import org.apache.orc.StripeInformation;
/**
* Value for OrcFileMergeMapper. Contains stripe related information for the
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 0567cbe..b3b48d5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -39,6 +39,23 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.codec.binary.Hex;
+import org.apache.hadoop.hive.ql.io.IOConstants;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.FileMetaInfo;
+import org.apache.orc.OrcUtils;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
+import org.apache.orc.TypeDescription;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -63,8 +80,6 @@ import org.apache.hadoop.hive.ql.io.LlapWrappableInputFormatInterface;
import org.apache.hadoop.hive.ql.io.RecordIdentifier;
import org.apache.hadoop.hive.ql.io.SelfDescribingInputFormatInterface;
import org.apache.hadoop.hive.ql.io.StatsProvidingRecordReader;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.WriterVersion;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
@@ -86,6 +101,7 @@ import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.StringUtils;
+import org.apache.orc.OrcProto;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
@@ -234,7 +250,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
/**
* Do we have schema on read in the configuration variables?
*/
- TypeDescription schema = OrcUtils.getDesiredRowTypeDescr(conf);
+ TypeDescription schema = getDesiredRowTypeDescr(conf);
Reader.Options options = new Reader.Options().range(offset, length);
options.schema(schema);
@@ -242,7 +258,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
List<OrcProto.Type> types = file.getTypes();
options.include(genIncludedColumns(types, conf, isOriginal));
setSearchArgument(options, types, conf, isOriginal);
- return file.rowsOptions(options);
+ return (RecordReader) file.rowsOptions(options);
}
public static boolean isOriginal(Reader file) {
@@ -1500,7 +1516,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
/**
* Do we have schema on read in the configuration variables?
*/
- TypeDescription schema = OrcUtils.getDesiredRowTypeDescr(conf);
+ TypeDescription schema = getDesiredRowTypeDescr(conf);
final Reader reader;
final int bucket;
@@ -1508,7 +1524,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
readOptions.range(split.getStart(), split.getLength());
// TODO: Convert genIncludedColumns and setSearchArgument to use TypeDescription.
- final List<Type> schemaTypes = OrcUtils.getOrcTypes(schema);
+ final List<OrcProto.Type> schemaTypes = OrcUtils.getOrcTypes(schema);
readOptions.include(genIncludedColumns(schemaTypes, conf, SCHEMA_TYPES_IS_ORIGINAL));
setSearchArgument(readOptions, schemaTypes, conf, SCHEMA_TYPES_IS_ORIGINAL);
@@ -1594,7 +1610,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
}
public static boolean[] pickStripesViaTranslatedSarg(SearchArgument sarg,
- WriterVersion writerVersion, List<OrcProto.Type> types,
+ OrcFile.WriterVersion writerVersion, List<OrcProto.Type> types,
List<StripeStatistics> stripeStats, int stripeCount) {
LOG.info("Translated ORC pushdown predicate: " + sarg);
assert sarg != null;
@@ -1608,7 +1624,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
}
private static boolean[] pickStripes(SearchArgument sarg, String[] sargColNames,
- WriterVersion writerVersion, boolean isOriginal, List<StripeStatistics> stripeStats,
+ OrcFile.WriterVersion writerVersion, boolean isOriginal, List<StripeStatistics> stripeStats,
int stripeCount, Path filePath) {
if (sarg == null || stripeStats == null || writerVersion == OrcFile.WriterVersion.ORIGINAL) {
return null; // only do split pruning if HIVE-8732 has been fixed in the writer
@@ -1927,4 +1943,165 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
return true;
}
}
+ /**
+ * Convert a Hive type property string that contains separated type names into a list of
+ * TypeDescription objects.
+ * @return the list of TypeDescription objects.
+ */
+ public static ArrayList<TypeDescription> typeDescriptionsFromHiveTypeProperty(
+ String hiveTypeProperty) {
+
+ // CONSDIER: We need a type name parser for TypeDescription.
+
+ ArrayList<TypeInfo> typeInfoList = TypeInfoUtils.getTypeInfosFromTypeString(hiveTypeProperty);
+ ArrayList<TypeDescription> typeDescrList =new ArrayList<TypeDescription>(typeInfoList.size());
+ for (TypeInfo typeInfo : typeInfoList) {
+ typeDescrList.add(convertTypeInfo(typeInfo));
+ }
+ return typeDescrList;
+ }
+
+ public static TypeDescription convertTypeInfo(TypeInfo info) {
+ switch (info.getCategory()) {
+ case PRIMITIVE: {
+ PrimitiveTypeInfo pinfo = (PrimitiveTypeInfo) info;
+ switch (pinfo.getPrimitiveCategory()) {
+ case BOOLEAN:
+ return TypeDescription.createBoolean();
+ case BYTE:
+ return TypeDescription.createByte();
+ case SHORT:
+ return TypeDescription.createShort();
+ case INT:
+ return TypeDescription.createInt();
+ case LONG:
+ return TypeDescription.createLong();
+ case FLOAT:
+ return TypeDescription.createFloat();
+ case DOUBLE:
+ return TypeDescription.createDouble();
+ case STRING:
+ return TypeDescription.createString();
+ case DATE:
+ return TypeDescription.createDate();
+ case TIMESTAMP:
+ return TypeDescription.createTimestamp();
+ case BINARY:
+ return TypeDescription.createBinary();
+ case DECIMAL: {
+ DecimalTypeInfo dinfo = (DecimalTypeInfo) pinfo;
+ return TypeDescription.createDecimal()
+ .withScale(dinfo.getScale())
+ .withPrecision(dinfo.getPrecision());
+ }
+ case VARCHAR: {
+ BaseCharTypeInfo cinfo = (BaseCharTypeInfo) pinfo;
+ return TypeDescription.createVarchar()
+ .withMaxLength(cinfo.getLength());
+ }
+ case CHAR: {
+ BaseCharTypeInfo cinfo = (BaseCharTypeInfo) pinfo;
+ return TypeDescription.createChar()
+ .withMaxLength(cinfo.getLength());
+ }
+ default:
+ throw new IllegalArgumentException("ORC doesn't handle primitive" +
+ " category " + pinfo.getPrimitiveCategory());
+ }
+ }
+ case LIST: {
+ ListTypeInfo linfo = (ListTypeInfo) info;
+ return TypeDescription.createList
+ (convertTypeInfo(linfo.getListElementTypeInfo()));
+ }
+ case MAP: {
+ MapTypeInfo minfo = (MapTypeInfo) info;
+ return TypeDescription.createMap
+ (convertTypeInfo(minfo.getMapKeyTypeInfo()),
+ convertTypeInfo(minfo.getMapValueTypeInfo()));
+ }
+ case UNION: {
+ UnionTypeInfo minfo = (UnionTypeInfo) info;
+ TypeDescription result = TypeDescription.createUnion();
+ for (TypeInfo child: minfo.getAllUnionObjectTypeInfos()) {
+ result.addUnionChild(convertTypeInfo(child));
+ }
+ return result;
+ }
+ case STRUCT: {
+ StructTypeInfo sinfo = (StructTypeInfo) info;
+ TypeDescription result = TypeDescription.createStruct();
+ for(String fieldName: sinfo.getAllStructFieldNames()) {
+ result.addField(fieldName,
+ convertTypeInfo(sinfo.getStructFieldTypeInfo(fieldName)));
+ }
+ return result;
+ }
+ default:
+ throw new IllegalArgumentException("ORC doesn't handle " +
+ info.getCategory());
+ }
+ }
+
+
+ public static TypeDescription getDesiredRowTypeDescr(Configuration conf) {
+
+ String columnNameProperty = null;
+ String columnTypeProperty = null;
+
+ ArrayList<String> schemaEvolutionColumnNames = null;
+ ArrayList<TypeDescription> schemaEvolutionTypeDescrs = null;
+
+ boolean haveSchemaEvolutionProperties = false;
+ if (HiveConf.getBoolVar(conf, ConfVars.HIVE_SCHEMA_EVOLUTION)) {
+
+ columnNameProperty = conf.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS);
+ columnTypeProperty = conf.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES);
+
+ haveSchemaEvolutionProperties =
+ (columnNameProperty != null && columnTypeProperty != null);
+
+ if (haveSchemaEvolutionProperties) {
+ schemaEvolutionColumnNames = Lists.newArrayList(columnNameProperty.split(","));
+ if (schemaEvolutionColumnNames.size() == 0) {
+ haveSchemaEvolutionProperties = false;
+ } else {
+ schemaEvolutionTypeDescrs =
+ typeDescriptionsFromHiveTypeProperty(columnTypeProperty);
+ if (schemaEvolutionTypeDescrs.size() != schemaEvolutionColumnNames.size()) {
+ haveSchemaEvolutionProperties = false;
+ }
+ }
+ }
+ }
+
+ if (!haveSchemaEvolutionProperties) {
+
+ // Try regular properties;
+ columnNameProperty = conf.get(serdeConstants.LIST_COLUMNS);
+ columnTypeProperty = conf.get(serdeConstants.LIST_COLUMN_TYPES);
+ if (columnTypeProperty == null || columnNameProperty == null) {
+ return null;
+ }
+
+ schemaEvolutionColumnNames = Lists.newArrayList(columnNameProperty.split(","));
+ if (schemaEvolutionColumnNames.size() == 0) {
+ return null;
+ }
+ schemaEvolutionTypeDescrs =
+ typeDescriptionsFromHiveTypeProperty(columnTypeProperty);
+ if (schemaEvolutionTypeDescrs.size() != schemaEvolutionColumnNames.size()) {
+ return null;
+ }
+ }
+
+ // Desired schema does not include virtual columns or partition columns.
+ TypeDescription result = TypeDescription.createStruct();
+ for (int i = 0; i < schemaEvolutionColumnNames.size(); i++) {
+ result.addField(schemaEvolutionColumnNames.get(i), schemaEvolutionTypeDescrs.get(i));
+ }
+
+ return result;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
index c15b35f..2782d7e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.orc.OrcProto;
/** An InputFormat for ORC files. Keys are meaningless,
* value is the OrcStruct object */
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java
index edc7490..2e63aba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewSplit.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.io.AcidInputFormat;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.orc.FileMetaInfo;
/**
* OrcFileSplit. Holds file meta info
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
index 2d0eaaf..3fb6a86 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.Properties;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.TypeDescription;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileSystem;
@@ -33,7 +35,6 @@ import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.io.RecordUpdater;
import org.apache.hadoop.hive.ql.io.StatsProvidingRecordWriter;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.EncodingStrategy;
import org.apache.hadoop.hive.ql.io.orc.OrcSerde.OrcSerdeRow;
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -160,7 +161,7 @@ public class OrcOutputFormat extends FileOutputFormat<NullWritable, OrcSerdeRow>
TypeDescription schema = TypeDescription.createStruct();
for (int i = 0; i < columnNames.size(); ++i) {
schema.addField(columnNames.get(i),
- OrcUtils.convertTypeInfo(columnTypes.get(i)));
+ OrcInputFormat.convertTypeInfo(columnTypes.get(i)));
}
if (LOG.isDebugEnabled()) {
LOG.debug("ORC schema = " + schema);
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
index 090d63b..e5f9786 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
@@ -26,6 +26,11 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import org.apache.orc.OrcUtils;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.TypeDescription;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -442,7 +447,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
this.length = options.getLength();
this.validTxnList = validTxnList;
- TypeDescription typeDescr = OrcUtils.getDesiredRowTypeDescr(conf);
+ TypeDescription typeDescr = OrcInputFormat.getDesiredRowTypeDescr(conf);
if (typeDescr == null) {
throw new IOException(ErrorMsg.SCHEMA_REQUIRED_TO_READ_ACID_TABLES.getErrorCodedMsg());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
index c0e9b1a..59876e2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Properties;
+import org.apache.orc.OrcConf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
index 81afb48..61cde41 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
@@ -25,12 +25,12 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import org.apache.orc.FileMetaInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.ColumnarSplit;
import org.apache.hadoop.hive.ql.io.AcidInputFormat;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapred.FileSplit;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
index 7a17b92..d48cadd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
import org.apache.hadoop.io.Writable;
+import org.apache.orc.OrcProto;
import java.io.DataInput;
import java.io.DataOutput;
[02/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
index c3916d3..993be71 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
@@ -22,7 +22,6 @@ import static com.google.common.base.Preconditions.checkArgument;
import java.io.IOException;
import java.io.OutputStream;
-import java.lang.management.ManagementFactory;
import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.ArrayList;
@@ -33,6 +32,30 @@ import java.util.Map;
import java.util.TimeZone;
import java.util.TreeMap;
+import org.apache.orc.BinaryColumnStatistics;
+import org.apache.orc.impl.BitFieldWriter;
+import org.apache.orc.impl.ColumnStatisticsImpl;
+import org.apache.orc.CompressionCodec;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.impl.DynamicIntArray;
+import org.apache.orc.impl.IntegerWriter;
+import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.OrcConf;
+import org.apache.orc.OrcUtils;
+import org.apache.orc.impl.OutStream;
+import org.apache.orc.impl.PositionRecorder;
+import org.apache.orc.impl.PositionedOutputStream;
+import org.apache.orc.impl.RunLengthByteWriter;
+import org.apache.orc.impl.RunLengthIntegerWriter;
+import org.apache.orc.impl.RunLengthIntegerWriterV2;
+import org.apache.orc.impl.SerializationUtils;
+import org.apache.orc.impl.SnappyCodec;
+import org.apache.orc.impl.StreamName;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.impl.StringRedBlackTree;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.impl.ZlibCodec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -52,13 +75,7 @@ import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.filters.BloomFilterIO;
-import org.apache.hadoop.hive.ql.io.orc.CompressionCodec.Modifier;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.CompressionStrategy;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.EncodingStrategy;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem;
+import org.apache.orc.CompressionCodec.Modifier;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
@@ -85,9 +102,9 @@ import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
+import org.apache.orc.OrcProto;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.primitives.Longs;
import com.google.protobuf.ByteString;
@@ -182,8 +199,8 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
boolean addBlockPadding,
OrcFile.Version version,
OrcFile.WriterCallback callback,
- EncodingStrategy encodingStrategy,
- CompressionStrategy compressionStrategy,
+ OrcFile.EncodingStrategy encodingStrategy,
+ OrcFile.CompressionStrategy compressionStrategy,
double paddingTolerance,
long blockSizeValue,
String bloomFilterColumnNames,
@@ -468,7 +485,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
case BLOOM_FILTER:
case DATA:
case DICTIONARY_DATA:
- if (getCompressionStrategy() == CompressionStrategy.SPEED) {
+ if (getCompressionStrategy() == OrcFile.CompressionStrategy.SPEED) {
modifiers = EnumSet.of(Modifier.FAST, Modifier.TEXT);
} else {
modifiers = EnumSet.of(Modifier.DEFAULT, Modifier.TEXT);
@@ -532,7 +549,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
* Get the encoding strategy to use.
* @return encoding strategy
*/
- public EncodingStrategy getEncodingStrategy() {
+ public OrcFile.EncodingStrategy getEncodingStrategy() {
return encodingStrategy;
}
@@ -540,7 +557,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
* Get the compression strategy to use.
* @return compression strategy
*/
- public CompressionStrategy getCompressionStrategy() {
+ public OrcFile.CompressionStrategy getCompressionStrategy() {
return compressionStrategy;
}
@@ -610,7 +627,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
private final OrcProto.BloomFilter.Builder bloomFilterEntry;
private boolean foundNulls;
private OutStream isPresentOutStream;
- private final List<StripeStatistics.Builder> stripeStatsBuilders;
+ private final List<OrcProto.StripeStatistics.Builder> stripeStatsBuilders;
private final StreamFactory streamFactory;
/**
@@ -687,7 +704,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
StreamFactory writer) {
if (isDirectV2) {
boolean alignedBitpacking = false;
- if (writer.getEncodingStrategy().equals(EncodingStrategy.SPEED)) {
+ if (writer.getEncodingStrategy().equals(OrcFile.EncodingStrategy.SPEED)) {
alignedBitpacking = true;
}
return new RunLengthIntegerWriterV2(output, signed, alignedBitpacking);
@@ -788,7 +805,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
private void removeIsPresentPositions() {
for(int i=0; i < rowIndex.getEntryCount(); ++i) {
- RowIndexEntry.Builder entry = rowIndex.getEntryBuilder(i);
+ OrcProto.RowIndexEntry.Builder entry = rowIndex.getEntryBuilder(i);
List<Long> positions = entry.getPositionsList();
// bit streams use 3 positions if uncompressed, 4 if compressed
positions = positions.subList(isCompressed ? 4 : 3, positions.size());
@@ -2772,7 +2789,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
type.setMaximumLength(schema.getMaxLength());
break;
case VARCHAR:
- type.setKind(Type.Kind.VARCHAR);
+ type.setKind(OrcProto.Type.Kind.VARCHAR);
type.setMaximumLength(schema.getMaxLength());
break;
case BINARY:
@@ -3284,9 +3301,9 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
}
@Override
- public void appendUserMetadata(List<UserMetadataItem> userMetadata) {
+ public void appendUserMetadata(List<OrcProto.UserMetadataItem> userMetadata) {
if (userMetadata != null) {
- for (UserMetadataItem item : userMetadata) {
+ for (OrcProto.UserMetadataItem item : userMetadata) {
this.userMetadata.put(item.getName(), item.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java
deleted file mode 100644
index 03cc3c5..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.EnumSet;
-import java.util.zip.DataFormatException;
-import java.util.zip.Deflater;
-import java.util.zip.Inflater;
-
-import javax.annotation.Nullable;
-
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectCompressionType;
-import org.apache.hadoop.hive.shims.HadoopShims.DirectDecompressorShim;
-import org.apache.hadoop.hive.shims.ShimLoader;
-
-class ZlibCodec implements CompressionCodec, DirectDecompressionCodec {
-
- private Boolean direct = null;
-
- private final int level;
- private final int strategy;
-
- public ZlibCodec() {
- level = Deflater.DEFAULT_COMPRESSION;
- strategy = Deflater.DEFAULT_STRATEGY;
- }
-
- private ZlibCodec(int level, int strategy) {
- this.level = level;
- this.strategy = strategy;
- }
-
- @Override
- public boolean compress(ByteBuffer in, ByteBuffer out,
- ByteBuffer overflow) throws IOException {
- Deflater deflater = new Deflater(level, true);
- deflater.setStrategy(strategy);
- int length = in.remaining();
- deflater.setInput(in.array(), in.arrayOffset() + in.position(), length);
- deflater.finish();
- int outSize = 0;
- int offset = out.arrayOffset() + out.position();
- while (!deflater.finished() && (length > outSize)) {
- int size = deflater.deflate(out.array(), offset, out.remaining());
- out.position(size + out.position());
- outSize += size;
- offset += size;
- // if we run out of space in the out buffer, use the overflow
- if (out.remaining() == 0) {
- if (overflow == null) {
- deflater.end();
- return false;
- }
- out = overflow;
- offset = out.arrayOffset() + out.position();
- }
- }
- deflater.end();
- return length > outSize;
- }
-
- @Override
- public void decompress(ByteBuffer in, ByteBuffer out) throws IOException {
-
- if(in.isDirect() && out.isDirect()) {
- directDecompress(in, out);
- return;
- }
-
- Inflater inflater = new Inflater(true);
- inflater.setInput(in.array(), in.arrayOffset() + in.position(),
- in.remaining());
- while (!(inflater.finished() || inflater.needsDictionary() ||
- inflater.needsInput())) {
- try {
- int count = inflater.inflate(out.array(),
- out.arrayOffset() + out.position(),
- out.remaining());
- out.position(count + out.position());
- } catch (DataFormatException dfe) {
- throw new IOException("Bad compression data", dfe);
- }
- }
- out.flip();
- inflater.end();
- in.position(in.limit());
- }
-
- @Override
- public boolean isAvailable() {
- if (direct == null) {
- // see nowrap option in new Inflater(boolean) which disables zlib headers
- try {
- if (ShimLoader.getHadoopShims().getDirectDecompressor(
- DirectCompressionType.ZLIB_NOHEADER) != null) {
- direct = Boolean.valueOf(true);
- } else {
- direct = Boolean.valueOf(false);
- }
- } catch (UnsatisfiedLinkError ule) {
- direct = Boolean.valueOf(false);
- }
- }
- return direct.booleanValue();
- }
-
- @Override
- public void directDecompress(ByteBuffer in, ByteBuffer out)
- throws IOException {
- DirectDecompressorShim decompressShim = ShimLoader.getHadoopShims()
- .getDirectDecompressor(DirectCompressionType.ZLIB_NOHEADER);
- decompressShim.decompress(in, out);
- out.flip(); // flip for read
- }
-
- @Override
- public CompressionCodec modify(@Nullable EnumSet<Modifier> modifiers) {
-
- if (modifiers == null) {
- return this;
- }
-
- int l = this.level;
- int s = this.strategy;
-
- for (Modifier m : modifiers) {
- switch (m) {
- case BINARY:
- /* filtered == less LZ77, more huffman */
- s = Deflater.FILTERED;
- break;
- case TEXT:
- s = Deflater.DEFAULT_STRATEGY;
- break;
- case FASTEST:
- // deflate_fast looking for 8 byte patterns
- l = Deflater.BEST_SPEED;
- break;
- case FAST:
- // deflate_fast looking for 16 byte patterns
- l = Deflater.BEST_SPEED + 1;
- break;
- case DEFAULT:
- // deflate_slow looking for 128 byte patterns
- l = Deflater.DEFAULT_COMPRESSION;
- break;
- default:
- break;
- }
- }
- return new ZlibCodec(l, s);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java
index 0691050..96af96a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReader.java
@@ -20,11 +20,9 @@ package org.apache.hadoop.hive.ql.io.orc.encoded;
import java.io.IOException;
import java.util.List;
-import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
+import org.apache.orc.StripeInformation;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch;
+import org.apache.orc.OrcProto;
public interface EncodedReader {
@@ -41,7 +39,8 @@ public interface EncodedReader {
* @param consumer The sink for data that has been read.
*/
void readEncodedColumns(int stripeIx, StripeInformation stripe,
- RowIndex[] index, List<ColumnEncoding> encodings, List<Stream> streams,
+ OrcProto.RowIndex[] index, List<OrcProto.ColumnEncoding> encodings,
+ List<OrcProto.Stream> streams,
boolean[] included, boolean[][] colRgs,
Consumer<OrcEncodedColumnBatch> consumer) throws IOException;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index ea7e0fa..fed6de0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -34,24 +34,18 @@ import org.apache.hadoop.hive.common.io.DataCache.DiskRangeListFactory;
import org.apache.hadoop.hive.common.io.DiskRangeList.CreateHelper;
import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
-import org.apache.hadoop.hive.ql.io.orc.CompressionCodec;
-import org.apache.hadoop.hive.ql.io.orc.DataReader;
-import org.apache.hadoop.hive.ql.io.orc.OrcConf;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto;
-import org.apache.hadoop.hive.ql.io.orc.OutStream;
+import org.apache.orc.CompressionCodec;
+import org.apache.orc.DataReader;
+import org.apache.orc.OrcConf;
+import org.apache.orc.impl.OutStream;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils;
-import org.apache.hadoop.hive.ql.io.orc.StreamName;
-import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
+import org.apache.orc.impl.StreamName;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.impl.BufferChunk;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils.ByteBufferAllocatorPool;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.PoolFactory;
-
-
+import org.apache.orc.OrcProto;
/**
* Encoded reader implementation.
@@ -135,7 +129,8 @@ class EncodedReaderImpl implements EncodedReader {
/** Helper context for each column being read */
private static final class ColumnReadContext {
- public ColumnReadContext(int colIx, ColumnEncoding encoding, RowIndex rowIndex) {
+ public ColumnReadContext(int colIx, OrcProto.ColumnEncoding encoding,
+ OrcProto.RowIndex rowIndex) {
this.encoding = encoding;
this.rowIndex = rowIndex;
this.colIx = colIx;
@@ -147,7 +142,7 @@ class EncodedReaderImpl implements EncodedReader {
int streamCount = 0;
final StreamContext[] streams = new StreamContext[MAX_STREAMS];
/** Column encoding. */
- ColumnEncoding encoding;
+ OrcProto.ColumnEncoding encoding;
/** Column rowindex. */
OrcProto.RowIndex rowIndex;
/** Column index in the file. */
@@ -204,7 +199,7 @@ class EncodedReaderImpl implements EncodedReader {
@Override
public void readEncodedColumns(int stripeIx, StripeInformation stripe,
- RowIndex[] indexes, List<ColumnEncoding> encodings, List<Stream> streamList,
+ OrcProto.RowIndex[] indexes, List<OrcProto.ColumnEncoding> encodings, List<OrcProto.Stream> streamList,
boolean[] included, boolean[][] colRgs,
Consumer<OrcEncodedColumnBatch> consumer) throws IOException {
// Note: for now we don't have to setError here, caller will setError if we throw.
@@ -351,7 +346,7 @@ class EncodedReaderImpl implements EncodedReader {
continue; // TODO: this would be invalid with HL cache, where RG x col can be excluded.
}
ColumnReadContext ctx = colCtxs[colIxMod];
- RowIndexEntry index = ctx.rowIndex.getEntry(rgIx),
+ OrcProto.RowIndexEntry index = ctx.rowIndex.getEntry(rgIx),
nextIndex = isLastRg ? null : ctx.rowIndex.getEntry(rgIx + 1);
ecb.initColumn(colIxMod, ctx.colIx, OrcEncodedColumnBatch.MAX_DATA_STREAMS);
for (int streamIx = 0; streamIx < ctx.streamCount; ++streamIx) {
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
index dd6f64c..fe46446 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
@@ -22,12 +22,11 @@ import java.util.List;
import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch;
import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
-import org.apache.hadoop.hive.ql.io.orc.CompressionCodec;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind;
-import org.apache.hadoop.hive.ql.io.orc.PositionProvider;
-import org.apache.hadoop.hive.ql.io.orc.SettableUncompressedStream;
+import org.apache.orc.CompressionCodec;
+import org.apache.orc.impl.PositionProvider;
+import org.apache.orc.impl.SettableUncompressedStream;
import org.apache.hadoop.hive.ql.io.orc.TreeReaderFactory;
+import org.apache.orc.OrcProto;
public class EncodedTreeReaderFactory extends TreeReaderFactory {
/**
@@ -87,13 +86,13 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_secondsStream != null) {
- _secondsStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _secondsStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
if (_nanosStream != null) {
- _nanosStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.SECONDARY_VALUE]));
+ _nanosStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.SECONDARY_VALUE]));
}
}
@@ -232,24 +231,24 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
if (!_isDictionaryEncoding) {
if (_lengthStream != null) {
- _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.LENGTH_VALUE]));
+ _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE]));
}
}
// set these streams only if the stripe is different
if (!sameStripe && _isDictionaryEncoding) {
if (_lengthStream != null) {
- _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.LENGTH_VALUE]));
+ _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE]));
}
if (_dictionaryStream != null) {
- _dictionaryStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DICTIONARY_DATA_VALUE]));
+ _dictionaryStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE]));
}
}
}
@@ -364,10 +363,10 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
}
@@ -461,10 +460,10 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
}
@@ -564,10 +563,10 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
}
@@ -661,10 +660,10 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
}
@@ -751,10 +750,10 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
}
@@ -852,13 +851,13 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_valueStream != null) {
- _valueStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _valueStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
if (_scaleStream != null) {
- _scaleStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.SECONDARY_VALUE]));
+ _scaleStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.SECONDARY_VALUE]));
}
}
@@ -972,10 +971,10 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
}
@@ -1099,24 +1098,24 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
if (!_isDictionaryEncoding) {
if (_lengthStream != null) {
- _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.LENGTH_VALUE]));
+ _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE]));
}
}
// set these streams only if the stripe is different
if (!sameStripe && _isDictionaryEncoding) {
if (_lengthStream != null) {
- _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.LENGTH_VALUE]));
+ _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE]));
}
if (_dictionaryStream != null) {
- _dictionaryStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DICTIONARY_DATA_VALUE]));
+ _dictionaryStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE]));
}
}
}
@@ -1267,24 +1266,24 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
if (!_isDictionaryEncoding) {
if (_lengthStream != null) {
- _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.LENGTH_VALUE]));
+ _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE]));
}
}
// set these streams only if the stripe is different
if (!sameStripe && _isDictionaryEncoding) {
if (_lengthStream != null) {
- _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.LENGTH_VALUE]));
+ _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE]));
}
if (_dictionaryStream != null) {
- _dictionaryStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DICTIONARY_DATA_VALUE]));
+ _dictionaryStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE]));
}
}
}
@@ -1404,10 +1403,10 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
}
@@ -1504,13 +1503,13 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
if (_lengthsStream != null) {
- _lengthsStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.LENGTH_VALUE]));
+ _lengthsStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE]));
}
}
@@ -1610,10 +1609,10 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
public void setBuffers(ColumnStreamData[] streamsData, boolean sameStripe)
throws IOException {
if (_presentStream != null) {
- _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.PRESENT_VALUE]));
+ _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE]));
}
if (_dataStream != null) {
- _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[Kind.DATA_VALUE]));
+ _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE]));
}
}
@@ -1683,11 +1682,11 @@ public class EncodedTreeReaderFactory extends TreeReaderFactory {
OrcProto.ColumnEncoding columnEncoding = encodings.get(columnIndex);
// stream buffers are arranged in enum order of stream kind
- ColumnStreamData present = streamBuffers[Kind.PRESENT_VALUE],
- data = streamBuffers[Kind.DATA_VALUE],
- dictionary = streamBuffers[Kind.DICTIONARY_DATA_VALUE],
- lengths = streamBuffers[Kind.LENGTH_VALUE],
- secondary = streamBuffers[Kind.SECONDARY_VALUE];
+ ColumnStreamData present = streamBuffers[OrcProto.Stream.Kind.PRESENT_VALUE],
+ data = streamBuffers[OrcProto.Stream.Kind.DATA_VALUE],
+ dictionary = streamBuffers[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE],
+ lengths = streamBuffers[OrcProto.Stream.Kind.LENGTH_VALUE],
+ secondary = streamBuffers[OrcProto.Stream.Kind.SECONDARY_VALUE];
switch (columnType.getKind()) {
case BINARY:
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java
index a27b35e..246ead6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.hive.common.Pool.PoolObjectHelper;
import org.apache.hadoop.hive.common.io.DataCache;
import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch;
import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
-import org.apache.hadoop.hive.ql.io.orc.DataReader;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
+import org.apache.orc.DataReader;
+import org.apache.orc.OrcProto;
/**
* The interface for reading encoded data from ORC files.
@@ -48,7 +48,7 @@ public interface Reader extends org.apache.hadoop.hive.ql.io.orc.Reader {
* All the previous streams are data streams, this and the next ones are index streams.
* We assume the sort will stay the same for backward compat.
*/
- public static final int MAX_DATA_STREAMS = Stream.Kind.ROW_INDEX.getNumber();
+ public static final int MAX_DATA_STREAMS = OrcProto.Stream.Kind.ROW_INDEX.getNumber();
public void init(long fileId, int stripeIx, int rgIx, int columnCount) {
if (batchKey == null) {
batchKey = new OrcBatchKey(fileId, stripeIx, rgIx);
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java
index a4fa03b..b0ac503 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java
@@ -22,9 +22,8 @@ import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.io.DataCache;
-import org.apache.hadoop.hive.ql.io.orc.DataReader;
+import org.apache.orc.DataReader;
import org.apache.hadoop.hive.ql.io.orc.OrcFile.ReaderOptions;
-import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReaderImpl;
class ReaderImpl extends org.apache.hadoop.hive.ql.io.orc.ReaderImpl implements Reader {
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/StreamUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/StreamUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/StreamUtils.java
index 2c9acfb..9ac53af 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/StreamUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/StreamUtils.java
@@ -23,9 +23,8 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.hive.common.DiskRangeInfo;
import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData;
import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
-import org.apache.hadoop.hive.ql.io.orc.SettableUncompressedStream;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
+import org.apache.orc.impl.SettableUncompressedStream;
+import org.apache.orc.impl.BufferChunk;
/**
* Stream utility.
@@ -63,7 +62,7 @@ public class StreamUtils {
// TODO: we should get rid of this
for (MemoryBuffer memoryBuffer : streamBuffer.getCacheBuffers()) {
ByteBuffer buffer = memoryBuffer.getByteBufferDup();
- diskRangeInfo.addDiskRange(new RecordReaderImpl.BufferChunk(buffer, offset));
+ diskRangeInfo.addDiskRange(new BufferChunk(buffer, offset));
offset += buffer.remaining();
}
return diskRangeInfo;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
index 42ad04b..7cddcc9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
@@ -28,9 +28,8 @@ import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto;
import org.apache.hadoop.hive.ql.io.orc.ReaderImpl;
-import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
+import org.apache.orc.StripeInformation;
import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -38,6 +37,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.orc.OrcProto;
/**
* The basic implementation of PartitionExpressionProxy that uses ql package classes.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
----------------------------------------------------------------------
diff --git a/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto b/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
deleted file mode 100644
index acadef9..0000000
--- a/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
+++ /dev/null
@@ -1,220 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package orc.proto;
-
-option java_package = "org.apache.hadoop.hive.ql.io.orc";
-
-message IntegerStatistics {
- optional sint64 minimum = 1;
- optional sint64 maximum = 2;
- optional sint64 sum = 3;
-}
-
-message DoubleStatistics {
- optional double minimum = 1;
- optional double maximum = 2;
- optional double sum = 3;
-}
-
-message StringStatistics {
- optional string minimum = 1;
- optional string maximum = 2;
- // sum will store the total length of all strings in a stripe
- optional sint64 sum = 3;
-}
-
-message BucketStatistics {
- repeated uint64 count = 1 [packed=true];
-}
-
-message DecimalStatistics {
- optional string minimum = 1;
- optional string maximum = 2;
- optional string sum = 3;
-}
-
-message DateStatistics {
- // min,max values saved as days since epoch
- optional sint32 minimum = 1;
- optional sint32 maximum = 2;
-}
-
-message TimestampStatistics {
- // min,max values saved as milliseconds since epoch
- optional sint64 minimum = 1;
- optional sint64 maximum = 2;
-}
-
-message BinaryStatistics {
- // sum will store the total binary blob length in a stripe
- optional sint64 sum = 1;
-}
-
-message ColumnStatistics {
- optional uint64 numberOfValues = 1;
- optional IntegerStatistics intStatistics = 2;
- optional DoubleStatistics doubleStatistics = 3;
- optional StringStatistics stringStatistics = 4;
- optional BucketStatistics bucketStatistics = 5;
- optional DecimalStatistics decimalStatistics = 6;
- optional DateStatistics dateStatistics = 7;
- optional BinaryStatistics binaryStatistics = 8;
- optional TimestampStatistics timestampStatistics = 9;
- optional bool hasNull = 10;
-}
-
-message RowIndexEntry {
- repeated uint64 positions = 1 [packed=true];
- optional ColumnStatistics statistics = 2;
-}
-
-message RowIndex {
- repeated RowIndexEntry entry = 1;
-}
-
-message BloomFilter {
- optional uint32 numHashFunctions = 1;
- repeated fixed64 bitset = 2;
-}
-
-message BloomFilterIndex {
- repeated BloomFilter bloomFilter = 1;
-}
-
-message Stream {
- // if you add new index stream kinds, you need to make sure to update
- // StreamName to ensure it is added to the stripe in the right area
- enum Kind {
- PRESENT = 0;
- DATA = 1;
- LENGTH = 2;
- DICTIONARY_DATA = 3;
- DICTIONARY_COUNT = 4;
- SECONDARY = 5;
- ROW_INDEX = 6;
- BLOOM_FILTER = 7;
- }
- optional Kind kind = 1;
- optional uint32 column = 2;
- optional uint64 length = 3;
-}
-
-message ColumnEncoding {
- enum Kind {
- DIRECT = 0;
- DICTIONARY = 1;
- DIRECT_V2 = 2;
- DICTIONARY_V2 = 3;
- }
- optional Kind kind = 1;
- optional uint32 dictionarySize = 2;
-}
-
-message StripeFooter {
- repeated Stream streams = 1;
- repeated ColumnEncoding columns = 2;
- optional string writerTimezone = 3;
-}
-
-message Type {
- enum Kind {
- BOOLEAN = 0;
- BYTE = 1;
- SHORT = 2;
- INT = 3;
- LONG = 4;
- FLOAT = 5;
- DOUBLE = 6;
- STRING = 7;
- BINARY = 8;
- TIMESTAMP = 9;
- LIST = 10;
- MAP = 11;
- STRUCT = 12;
- UNION = 13;
- DECIMAL = 14;
- DATE = 15;
- VARCHAR = 16;
- CHAR = 17;
- }
- optional Kind kind = 1;
- repeated uint32 subtypes = 2 [packed=true];
- repeated string fieldNames = 3;
- optional uint32 maximumLength = 4;
- optional uint32 precision = 5;
- optional uint32 scale = 6;
-}
-
-message StripeInformation {
- optional uint64 offset = 1;
- optional uint64 indexLength = 2;
- optional uint64 dataLength = 3;
- optional uint64 footerLength = 4;
- optional uint64 numberOfRows = 5;
-}
-
-message UserMetadataItem {
- optional string name = 1;
- optional bytes value = 2;
-}
-
-message StripeStatistics {
- repeated ColumnStatistics colStats = 1;
-}
-
-message Metadata {
- repeated StripeStatistics stripeStats = 1;
-}
-
-message Footer {
- optional uint64 headerLength = 1;
- optional uint64 contentLength = 2;
- repeated StripeInformation stripes = 3;
- repeated Type types = 4;
- repeated UserMetadataItem metadata = 5;
- optional uint64 numberOfRows = 6;
- repeated ColumnStatistics statistics = 7;
- optional uint32 rowIndexStride = 8;
-}
-
-enum CompressionKind {
- NONE = 0;
- ZLIB = 1;
- SNAPPY = 2;
- LZO = 3;
-}
-
-// Serialized length must be less that 255 bytes
-message PostScript {
- optional uint64 footerLength = 1;
- optional CompressionKind compression = 2;
- optional uint64 compressionBlockSize = 3;
- // the version of the file format
- // [0, 11] = Hive 0.11
- // [0, 12] = Hive 0.12
- repeated uint32 version = 4 [packed = true];
- optional uint64 metadataLength = 5;
- // Version of the writer:
- // 0 (or missing) = original
- // 1 = HIVE-8732 fixed
- // 2 = HIVE-4243 fixed
- optional uint32 writerVersion = 6;
- // Leave this last in the record
- optional string magic = 8000;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java
index a95fee9..e29ad7a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java
@@ -31,9 +31,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
+import org.apache.hadoop.hive.ql.io.orc.Writer;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.TestVectorizedORCReader;
-import org.apache.hadoop.hive.ql.io.orc.Writer;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java
deleted file mode 100644
index b537765..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-
-public class TestBitFieldReader {
-
- public void runSeekTest(CompressionCodec codec) throws Exception {
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- final int COUNT = 16384;
- BitFieldWriter out = new BitFieldWriter(
- new OutStream("test", 500, codec, collect), 1);
- TestInStream.PositionCollector[] positions =
- new TestInStream.PositionCollector[COUNT];
- for(int i=0; i < COUNT; ++i) {
- positions[i] = new TestInStream.PositionCollector();
- out.getPosition(positions[i]);
- // test runs, non-runs
- if (i < COUNT / 2) {
- out.write(i & 1);
- } else {
- out.write((i/3) & 1);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- BitFieldReader in = new BitFieldReader(InStream.create("test",
- new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
- codec, 500), 1);
- for(int i=0; i < COUNT; ++i) {
- int x = in.next();
- if (i < COUNT / 2) {
- assertEquals(i & 1, x);
- } else {
- assertEquals((i/3) & 1, x);
- }
- }
- for(int i=COUNT-1; i >= 0; --i) {
- in.seek(positions[i]);
- int x = in.next();
- if (i < COUNT / 2) {
- assertEquals(i & 1, x);
- } else {
- assertEquals((i/3) & 1, x);
- }
- }
- }
-
- @Test
- public void testUncompressedSeek() throws Exception {
- runSeekTest(null);
- }
-
- @Test
- public void testCompressedSeek() throws Exception {
- runSeekTest(new ZlibCodec());
- }
-
- @Test
- public void testBiggerItems() throws Exception {
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- final int COUNT = 16384;
- BitFieldWriter out = new BitFieldWriter(
- new OutStream("test", 500, null, collect), 3);
- for(int i=0; i < COUNT; ++i) {
- // test runs, non-runs
- if (i < COUNT / 2) {
- out.write(i & 7);
- } else {
- out.write((i/3) & 7);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- BitFieldReader in = new BitFieldReader(InStream.create("test",
- new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
- null, 500), 3);
- for(int i=0; i < COUNT; ++i) {
- int x = in.next();
- if (i < COUNT / 2) {
- assertEquals(i & 7, x);
- } else {
- assertEquals((i/3) & 7, x);
- }
- }
- }
-
- @Test
- public void testSkips() throws Exception {
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- BitFieldWriter out = new BitFieldWriter(
- new OutStream("test", 100, null, collect), 1);
- final int COUNT = 16384;
- for(int i=0; i < COUNT; ++i) {
- if (i < COUNT/2) {
- out.write(i & 1);
- } else {
- out.write((i/3) & 1);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- BitFieldReader in = new BitFieldReader(InStream.create("test", new ByteBuffer[]{inBuf},
- new long[]{0}, inBuf.remaining(), null, 100), 1);
- for(int i=0; i < COUNT; i += 5) {
- int x = (int) in.next();
- if (i < COUNT/2) {
- assertEquals(i & 1, x);
- } else {
- assertEquals((i/3) & 1, x);
- }
- if (i < COUNT - 5) {
- in.skip(4);
- }
- in.skip(0);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitPack.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitPack.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitPack.java
deleted file mode 100644
index 41a807b..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitPack.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.io.LongWritable;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Longs;
-
-public class TestBitPack {
-
- private static final int SIZE = 100;
- private static Random rand = new Random(100);
- Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test"
- + File.separator + "tmp"));
-
- Configuration conf;
- FileSystem fs;
- Path testFilePath;
-
- @Rule
- public TestName testCaseName = new TestName();
-
- @Before
- public void openFileSystem() throws Exception {
- conf = new Configuration();
- fs = FileSystem.getLocal(conf);
- testFilePath = new Path(workDir, "TestOrcFile." + testCaseName.getMethodName() + ".orc");
- fs.delete(testFilePath, false);
- }
-
- private long[] deltaEncode(long[] inp) {
- long[] output = new long[inp.length];
- SerializationUtils utils = new SerializationUtils();
- for (int i = 0; i < inp.length; i++) {
- output[i] = utils.zigzagEncode(inp[i]);
- }
- return output;
- }
-
- private long nextLong(Random rng, long n) {
- long bits, val;
- do {
- bits = (rng.nextLong() << 1) >>> 1;
- val = bits % n;
- } while (bits - val + (n - 1) < 0L);
- return val;
- }
-
- private void runTest(int numBits) throws IOException {
- long[] inp = new long[SIZE];
- for (int i = 0; i < SIZE; i++) {
- long val = 0;
- if (numBits <= 32) {
- if (numBits == 1) {
- val = -1 * rand.nextInt(2);
- } else {
- val = rand.nextInt((int) Math.pow(2, numBits - 1));
- }
- } else {
- val = nextLong(rand, (long) Math.pow(2, numBits - 2));
- }
- if (val % 2 == 0) {
- val = -val;
- }
- inp[i] = val;
- }
- long[] deltaEncoded = deltaEncode(inp);
- long minInput = Collections.min(Longs.asList(deltaEncoded));
- long maxInput = Collections.max(Longs.asList(deltaEncoded));
- long rangeInput = maxInput - minInput;
- SerializationUtils utils = new SerializationUtils();
- int fixedWidth = utils.findClosestNumBits(rangeInput);
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- OutStream output = new OutStream("test", SIZE, null, collect);
- utils.writeInts(deltaEncoded, 0, deltaEncoded.length, fixedWidth, output);
- output.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- long[] buff = new long[SIZE];
- utils.readInts(buff, 0, SIZE, fixedWidth, InStream.create("test", new ByteBuffer[] { inBuf },
- new long[] { 0 }, inBuf.remaining(), null, SIZE));
- for (int i = 0; i < SIZE; i++) {
- buff[i] = utils.zigzagDecode(buff[i]);
- }
- assertEquals(numBits, fixedWidth);
- assertArrayEquals(inp, buff);
- }
-
- @Test
- public void test01BitPacking1Bit() throws IOException {
- runTest(1);
- }
-
- @Test
- public void test02BitPacking2Bit() throws IOException {
- runTest(2);
- }
-
- @Test
- public void test03BitPacking3Bit() throws IOException {
- runTest(3);
- }
-
- @Test
- public void test04BitPacking4Bit() throws IOException {
- runTest(4);
- }
-
- @Test
- public void test05BitPacking5Bit() throws IOException {
- runTest(5);
- }
-
- @Test
- public void test06BitPacking6Bit() throws IOException {
- runTest(6);
- }
-
- @Test
- public void test07BitPacking7Bit() throws IOException {
- runTest(7);
- }
-
- @Test
- public void test08BitPacking8Bit() throws IOException {
- runTest(8);
- }
-
- @Test
- public void test09BitPacking9Bit() throws IOException {
- runTest(9);
- }
-
- @Test
- public void test10BitPacking10Bit() throws IOException {
- runTest(10);
- }
-
- @Test
- public void test11BitPacking11Bit() throws IOException {
- runTest(11);
- }
-
- @Test
- public void test12BitPacking12Bit() throws IOException {
- runTest(12);
- }
-
- @Test
- public void test13BitPacking13Bit() throws IOException {
- runTest(13);
- }
-
- @Test
- public void test14BitPacking14Bit() throws IOException {
- runTest(14);
- }
-
- @Test
- public void test15BitPacking15Bit() throws IOException {
- runTest(15);
- }
-
- @Test
- public void test16BitPacking16Bit() throws IOException {
- runTest(16);
- }
-
- @Test
- public void test17BitPacking17Bit() throws IOException {
- runTest(17);
- }
-
- @Test
- public void test18BitPacking18Bit() throws IOException {
- runTest(18);
- }
-
- @Test
- public void test19BitPacking19Bit() throws IOException {
- runTest(19);
- }
-
- @Test
- public void test20BitPacking20Bit() throws IOException {
- runTest(20);
- }
-
- @Test
- public void test21BitPacking21Bit() throws IOException {
- runTest(21);
- }
-
- @Test
- public void test22BitPacking22Bit() throws IOException {
- runTest(22);
- }
-
- @Test
- public void test23BitPacking23Bit() throws IOException {
- runTest(23);
- }
-
- @Test
- public void test24BitPacking24Bit() throws IOException {
- runTest(24);
- }
-
- @Test
- public void test26BitPacking26Bit() throws IOException {
- runTest(26);
- }
-
- @Test
- public void test28BitPacking28Bit() throws IOException {
- runTest(28);
- }
-
- @Test
- public void test30BitPacking30Bit() throws IOException {
- runTest(30);
- }
-
- @Test
- public void test32BitPacking32Bit() throws IOException {
- runTest(32);
- }
-
- @Test
- public void test40BitPacking40Bit() throws IOException {
- runTest(40);
- }
-
- @Test
- public void test48BitPacking48Bit() throws IOException {
- runTest(48);
- }
-
- @Test
- public void test56BitPacking56Bit() throws IOException {
- runTest(56);
- }
-
- @Test
- public void test64BitPacking64Bit() throws IOException {
- runTest(64);
- }
-
- @Test
- public void testBitPack64Large() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- int size = 1080832;
- long[] inp = new long[size];
- Random rand = new Random(1234);
- for (int i = 0; i < size; i++) {
- inp[i] = rand.nextLong();
- }
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.ZLIB));
- for (Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
index a51177e..9433283 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java
@@ -33,9 +33,18 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.impl.ColumnStatisticsImpl;
+import org.apache.orc.DateColumnStatistics;
+import org.apache.orc.DecimalColumnStatistics;
+import org.apache.orc.DoubleColumnStatistics;
+import org.apache.orc.IntegerColumnStatistics;
+import org.apache.orc.StringColumnStatistics;
+import org.apache.orc.StripeStatistics;
+import org.apache.orc.TimestampColumnStatistics;
+import org.apache.orc.TypeDescription;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestDynamicArray.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestDynamicArray.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestDynamicArray.java
deleted file mode 100644
index c6d3c01..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestDynamicArray.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.util.Random;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class TestDynamicArray {
-
- @Test
- public void testByteArray() throws Exception {
- DynamicByteArray dba = new DynamicByteArray(3, 10);
- dba.add((byte) 0);
- dba.add((byte) 1);
- dba.set(3, (byte) 3);
- dba.set(2, (byte) 2);
- dba.add((byte) 4);
- assertEquals("{0,1,2,3,4}", dba.toString());
- assertEquals(5, dba.size());
- byte[] val;
- val = new byte[0];
- assertEquals(0, dba.compare(val, 0, 0, 2, 0));
- assertEquals(-1, dba.compare(val, 0, 0, 2, 1));
- val = new byte[]{3,42};
- assertEquals(1, dba.compare(val, 0, 1, 2, 0));
- assertEquals(1, dba.compare(val, 0, 1, 2, 1));
- assertEquals(0, dba.compare(val, 0, 1, 3, 1));
- assertEquals(-1, dba.compare(val, 0, 1, 3, 2));
- assertEquals(1, dba.compare(val, 0, 2, 3, 1));
- val = new byte[256];
- for(int b=-128; b < 128; ++b) {
- dba.add((byte) b);
- val[b+128] = (byte) b;
- }
- assertEquals(0, dba.compare(val, 0, 256, 5, 256));
- assertEquals(1, dba.compare(val, 0, 1, 0, 1));
- assertEquals(1, dba.compare(val, 254, 1, 0, 1));
- assertEquals(1, dba.compare(val, 120, 1, 64, 1));
- val = new byte[1024];
- Random rand = new Random(1701);
- for(int i = 0; i < val.length; ++i) {
- rand.nextBytes(val);
- }
- dba.add(val, 0, 1024);
- assertEquals(1285, dba.size());
- assertEquals(0, dba.compare(val, 0, 1024, 261, 1024));
- }
-
- @Test
- public void testIntArray() throws Exception {
- DynamicIntArray dia = new DynamicIntArray(10);
- for(int i=0; i < 10000; ++i) {
- dia.add(2*i);
- }
- assertEquals(10000, dia.size());
- for(int i=0; i < 10000; ++i) {
- assertEquals(2*i, dia.get(i));
- }
- dia.clear();
- assertEquals(0, dia.size());
- dia.add(3);
- dia.add(12);
- dia.add(65);
- assertEquals("{3,12,65}", dia.toString());
- for(int i=0; i < 5; ++i) {
- dia.increment(i, 3);
- }
- assertEquals("{6,15,68,3,3}", dia.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java
deleted file mode 100644
index 4c3ddfc..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.fail;
-
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Test;
-
-public class TestInStream {
-
- static class OutputCollector implements OutStream.OutputReceiver {
- DynamicByteArray buffer = new DynamicByteArray();
-
- @Override
- public void output(ByteBuffer buffer) throws IOException {
- this.buffer.add(buffer.array(), buffer.arrayOffset() + buffer.position(),
- buffer.remaining());
- }
- }
-
- static class PositionCollector
- implements PositionProvider, PositionRecorder {
- private List<Long> positions = new ArrayList<Long>();
- private int index = 0;
-
- @Override
- public long getNext() {
- return positions.get(index++);
- }
-
- @Override
- public void addPosition(long offset) {
- positions.add(offset);
- }
-
- public void reset() {
- index = 0;
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder("position: ");
- for(int i=0; i < positions.size(); ++i) {
- if (i != 0) {
- builder.append(", ");
- }
- builder.append(positions.get(i));
- }
- return builder.toString();
- }
- }
-
- @Test
- public void testUncompressed() throws Exception {
- OutputCollector collect = new OutputCollector();
- OutStream out = new OutStream("test", 100, null, collect);
- PositionCollector[] positions = new PositionCollector[1024];
- for(int i=0; i < 1024; ++i) {
- positions[i] = new PositionCollector();
- out.getPosition(positions[i]);
- out.write(i);
- }
- out.flush();
- assertEquals(1024, collect.buffer.size());
- for(int i=0; i < 1024; ++i) {
- assertEquals((byte) i, collect.buffer.get(i));
- }
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- InStream in = InStream.create("test", new ByteBuffer[]{inBuf},
- new long[]{0}, inBuf.remaining(), null, 100);
- assertEquals("uncompressed stream test position: 0 length: 1024" +
- " range: 0 offset: 0 limit: 0",
- in.toString());
- for(int i=0; i < 1024; ++i) {
- int x = in.read();
- assertEquals(i & 0xff, x);
- }
- for(int i=1023; i >= 0; --i) {
- in.seek(positions[i]);
- assertEquals(i & 0xff, in.read());
- }
- }
-
- @Test
- public void testCompressed() throws Exception {
- OutputCollector collect = new OutputCollector();
- CompressionCodec codec = new ZlibCodec();
- OutStream out = new OutStream("test", 300, codec, collect);
- PositionCollector[] positions = new PositionCollector[1024];
- for(int i=0; i < 1024; ++i) {
- positions[i] = new PositionCollector();
- out.getPosition(positions[i]);
- out.write(i);
- }
- out.flush();
- assertEquals("test", out.toString());
- assertEquals(961, collect.buffer.size());
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- InStream in = InStream.create("test", new ByteBuffer[]{inBuf},
- new long[]{0}, inBuf.remaining(), codec, 300);
- assertEquals("compressed stream test position: 0 length: 961 range: 0" +
- " offset: 0 limit: 0 range 0 = 0 to 961",
- in.toString());
- for(int i=0; i < 1024; ++i) {
- int x = in.read();
- assertEquals(i & 0xff, x);
- }
- assertEquals(0, in.available());
- for(int i=1023; i >= 0; --i) {
- in.seek(positions[i]);
- assertEquals(i & 0xff, in.read());
- }
- }
-
- @Test
- public void testCorruptStream() throws Exception {
- OutputCollector collect = new OutputCollector();
- CompressionCodec codec = new ZlibCodec();
- OutStream out = new OutStream("test", 500, codec, collect);
- PositionCollector[] positions = new PositionCollector[1024];
- for(int i=0; i < 1024; ++i) {
- positions[i] = new PositionCollector();
- out.getPosition(positions[i]);
- out.write(i);
- }
- out.flush();
-
- // now try to read the stream with a buffer that is too small
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- InStream in = InStream.create("test", new ByteBuffer[]{inBuf},
- new long[]{0}, inBuf.remaining(), codec, 100);
- byte[] contents = new byte[1024];
- try {
- in.read(contents);
- fail();
- } catch(IllegalArgumentException iae) {
- // EXPECTED
- }
-
- // make a corrupted header
- inBuf.clear();
- inBuf.put((byte) 32);
- inBuf.put((byte) 0);
- inBuf.flip();
- in = InStream.create("test2", new ByteBuffer[]{inBuf}, new long[]{0},
- inBuf.remaining(), codec, 300);
- try {
- in.read();
- fail();
- } catch (IllegalStateException ise) {
- // EXPECTED
- }
- }
-
- @Test
- public void testDisjointBuffers() throws Exception {
- OutputCollector collect = new OutputCollector();
- CompressionCodec codec = new ZlibCodec();
- OutStream out = new OutStream("test", 400, codec, collect);
- PositionCollector[] positions = new PositionCollector[1024];
- DataOutput stream = new DataOutputStream(out);
- for(int i=0; i < 1024; ++i) {
- positions[i] = new PositionCollector();
- out.getPosition(positions[i]);
- stream.writeInt(i);
- }
- out.flush();
- assertEquals("test", out.toString());
- assertEquals(1674, collect.buffer.size());
- ByteBuffer[] inBuf = new ByteBuffer[3];
- inBuf[0] = ByteBuffer.allocate(500);
- inBuf[1] = ByteBuffer.allocate(1200);
- inBuf[2] = ByteBuffer.allocate(500);
- collect.buffer.setByteBuffer(inBuf[0], 0, 483);
- collect.buffer.setByteBuffer(inBuf[1], 483, 1625 - 483);
- collect.buffer.setByteBuffer(inBuf[2], 1625, 1674 - 1625);
-
- for(int i=0; i < inBuf.length; ++i) {
- inBuf[i].flip();
- }
- InStream in = InStream.create("test", inBuf,
- new long[]{0,483, 1625}, 1674, codec, 400);
- assertEquals("compressed stream test position: 0 length: 1674 range: 0" +
- " offset: 0 limit: 0 range 0 = 0 to 483;" +
- " range 1 = 483 to 1142; range 2 = 1625 to 49",
- in.toString());
- DataInputStream inStream = new DataInputStream(in);
- for(int i=0; i < 1024; ++i) {
- int x = inStream.readInt();
- assertEquals(i, x);
- }
- assertEquals(0, in.available());
- for(int i=1023; i >= 0; --i) {
- in.seek(positions[i]);
- assertEquals(i, inStream.readInt());
- }
-
- in = InStream.create("test", new ByteBuffer[]{inBuf[1], inBuf[2]},
- new long[]{483, 1625}, 1674, codec, 400);
- inStream = new DataInputStream(in);
- positions[303].reset();
- in.seek(positions[303]);
- for(int i=303; i < 1024; ++i) {
- assertEquals(i, inStream.readInt());
- }
-
- in = InStream.create("test", new ByteBuffer[]{inBuf[0], inBuf[2]},
- new long[]{0, 1625}, 1674, codec, 400);
- inStream = new DataInputStream(in);
- positions[1001].reset();
- for(int i=0; i < 300; ++i) {
- assertEquals(i, inStream.readInt());
- }
- in.seek(positions[1001]);
- for(int i=1001; i < 1024; ++i) {
- assertEquals(i, inStream.readInt());
- }
- }
-
- @Test
- public void testUncompressedDisjointBuffers() throws Exception {
- OutputCollector collect = new OutputCollector();
- OutStream out = new OutStream("test", 400, null, collect);
- PositionCollector[] positions = new PositionCollector[1024];
- DataOutput stream = new DataOutputStream(out);
- for(int i=0; i < 1024; ++i) {
- positions[i] = new PositionCollector();
- out.getPosition(positions[i]);
- stream.writeInt(i);
- }
- out.flush();
- assertEquals("test", out.toString());
- assertEquals(4096, collect.buffer.size());
- ByteBuffer[] inBuf = new ByteBuffer[3];
- inBuf[0] = ByteBuffer.allocate(1100);
- inBuf[1] = ByteBuffer.allocate(2200);
- inBuf[2] = ByteBuffer.allocate(1100);
- collect.buffer.setByteBuffer(inBuf[0], 0, 1024);
- collect.buffer.setByteBuffer(inBuf[1], 1024, 2048);
- collect.buffer.setByteBuffer(inBuf[2], 3072, 1024);
-
- for(int i=0; i < inBuf.length; ++i) {
- inBuf[i].flip();
- }
- InStream in = InStream.create("test", inBuf,
- new long[]{0, 1024, 3072}, 4096, null, 400);
- assertEquals("uncompressed stream test position: 0 length: 4096" +
- " range: 0 offset: 0 limit: 0",
- in.toString());
- DataInputStream inStream = new DataInputStream(in);
- for(int i=0; i < 1024; ++i) {
- int x = inStream.readInt();
- assertEquals(i, x);
- }
- assertEquals(0, in.available());
- for(int i=1023; i >= 0; --i) {
- in.seek(positions[i]);
- assertEquals(i, inStream.readInt());
- }
-
- in = InStream.create("test", new ByteBuffer[]{inBuf[1], inBuf[2]},
- new long[]{1024, 3072}, 4096, null, 400);
- inStream = new DataInputStream(in);
- positions[256].reset();
- in.seek(positions[256]);
- for(int i=256; i < 1024; ++i) {
- assertEquals(i, inStream.readInt());
- }
-
- in = InStream.create("test", new ByteBuffer[]{inBuf[0], inBuf[2]},
- new long[]{0, 3072}, 4096, null, 400);
- inStream = new DataInputStream(in);
- positions[768].reset();
- for(int i=0; i < 256; ++i) {
- assertEquals(i, inStream.readInt());
- }
- in.seek(positions[768]);
- for(int i=768; i < 1024; ++i) {
- assertEquals(i, inStream.readInt());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index a4b948a..f81f5bb8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@ -108,6 +108,8 @@ import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
+import org.apache.orc.OrcProto;
+
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestIntegerCompressionReader.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestIntegerCompressionReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestIntegerCompressionReader.java
deleted file mode 100644
index 29dce30..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestIntegerCompressionReader.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-import org.junit.Test;
-
-public class TestIntegerCompressionReader {
-
- public void runSeekTest(CompressionCodec codec) throws Exception {
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- RunLengthIntegerWriterV2 out = new RunLengthIntegerWriterV2(
- new OutStream("test", 1000, codec, collect), true);
- TestInStream.PositionCollector[] positions =
- new TestInStream.PositionCollector[4096];
- Random random = new Random(99);
- int[] junk = new int[2048];
- for(int i=0; i < junk.length; ++i) {
- junk[i] = random.nextInt();
- }
- for(int i=0; i < 4096; ++i) {
- positions[i] = new TestInStream.PositionCollector();
- out.getPosition(positions[i]);
- // test runs, incrementing runs, non-runs
- if (i < 1024) {
- out.write(i/4);
- } else if (i < 2048) {
- out.write(2*i);
- } else {
- out.write(junk[i-2048]);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- RunLengthIntegerReaderV2 in =
- new RunLengthIntegerReaderV2(InStream.create
- ("test", new ByteBuffer[]{inBuf},
- new long[]{0}, inBuf.remaining(),
- codec, 1000), true, false);
- for(int i=0; i < 2048; ++i) {
- int x = (int) in.next();
- if (i < 1024) {
- assertEquals(i/4, x);
- } else if (i < 2048) {
- assertEquals(2*i, x);
- } else {
- assertEquals(junk[i-2048], x);
- }
- }
- for(int i=2047; i >= 0; --i) {
- in.seek(positions[i]);
- int x = (int) in.next();
- if (i < 1024) {
- assertEquals(i/4, x);
- } else if (i < 2048) {
- assertEquals(2*i, x);
- } else {
- assertEquals(junk[i-2048], x);
- }
- }
- }
-
- @Test
- public void testUncompressedSeek() throws Exception {
- runSeekTest(null);
- }
-
- @Test
- public void testCompressedSeek() throws Exception {
- runSeekTest(new ZlibCodec());
- }
-
- @Test
- public void testSkips() throws Exception {
- TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
- RunLengthIntegerWriterV2 out = new RunLengthIntegerWriterV2(
- new OutStream("test", 100, null, collect), true);
- for(int i=0; i < 2048; ++i) {
- if (i < 1024) {
- out.write(i);
- } else {
- out.write(256 * i);
- }
- }
- out.flush();
- ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
- collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
- inBuf.flip();
- RunLengthIntegerReaderV2 in =
- new RunLengthIntegerReaderV2(InStream.create("test",
- new ByteBuffer[]{inBuf},
- new long[]{0},
- inBuf.remaining(),
- null, 100), true, false);
- for(int i=0; i < 2048; i += 10) {
- int x = (int) in.next();
- if (i < 1024) {
- assertEquals(i, x);
- } else {
- assertEquals(256 * i, x);
- }
- if (i < 2038) {
- in.skip(9);
- }
- in.skip(0);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestJsonFileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestJsonFileDump.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestJsonFileDump.java
index d17c528..acf232d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestJsonFileDump.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestJsonFileDump.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hive.common.util.HiveTestUtils;
+import org.apache.orc.CompressionKind;
import org.junit.Before;
import org.junit.Test;
[15/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/gen/protobuf-java/org/apache/orc/OrcProto.java
----------------------------------------------------------------------
diff --git a/orc/src/gen/protobuf-java/org/apache/orc/OrcProto.java b/orc/src/gen/protobuf-java/org/apache/orc/OrcProto.java
new file mode 100644
index 0000000..b1577a3
--- /dev/null
+++ b/orc/src/gen/protobuf-java/org/apache/orc/OrcProto.java
@@ -0,0 +1,19279 @@
+// Generated by the protocol buffer compiler. DO NOT EDIT!
+// source: orc_proto.proto
+
+package org.apache.orc;
+
+public final class OrcProto {
+ private OrcProto() {}
+ public static void registerAllExtensions(
+ com.google.protobuf.ExtensionRegistry registry) {
+ }
+ /**
+ * Protobuf enum {@code orc.proto.CompressionKind}
+ */
+ public enum CompressionKind
+ implements com.google.protobuf.ProtocolMessageEnum {
+ /**
+ * <code>NONE = 0;</code>
+ */
+ NONE(0, 0),
+ /**
+ * <code>ZLIB = 1;</code>
+ */
+ ZLIB(1, 1),
+ /**
+ * <code>SNAPPY = 2;</code>
+ */
+ SNAPPY(2, 2),
+ /**
+ * <code>LZO = 3;</code>
+ */
+ LZO(3, 3),
+ ;
+
+ /**
+ * <code>NONE = 0;</code>
+ */
+ public static final int NONE_VALUE = 0;
+ /**
+ * <code>ZLIB = 1;</code>
+ */
+ public static final int ZLIB_VALUE = 1;
+ /**
+ * <code>SNAPPY = 2;</code>
+ */
+ public static final int SNAPPY_VALUE = 2;
+ /**
+ * <code>LZO = 3;</code>
+ */
+ public static final int LZO_VALUE = 3;
+
+
+ public final int getNumber() { return value; }
+
+ public static CompressionKind valueOf(int value) {
+ switch (value) {
+ case 0: return NONE;
+ case 1: return ZLIB;
+ case 2: return SNAPPY;
+ case 3: return LZO;
+ default: return null;
+ }
+ }
+
+ public static com.google.protobuf.Internal.EnumLiteMap<CompressionKind>
+ internalGetValueMap() {
+ return internalValueMap;
+ }
+ private static com.google.protobuf.Internal.EnumLiteMap<CompressionKind>
+ internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap<CompressionKind>() {
+ public CompressionKind findValueByNumber(int number) {
+ return CompressionKind.valueOf(number);
+ }
+ };
+
+ public final com.google.protobuf.Descriptors.EnumValueDescriptor
+ getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+ }
+ public final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptorForType() {
+ return getDescriptor();
+ }
+ public static final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.getDescriptor().getEnumTypes().get(0);
+ }
+
+ private static final CompressionKind[] VALUES = values();
+
+ public static CompressionKind valueOf(
+ com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+ throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+ }
+
+ private final int index;
+ private final int value;
+
+ private CompressionKind(int index, int value) {
+ this.index = index;
+ this.value = value;
+ }
+
+ // @@protoc_insertion_point(enum_scope:orc.proto.CompressionKind)
+ }
+
+ public interface IntegerStatisticsOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // optional sint64 minimum = 1;
+ /**
+ * <code>optional sint64 minimum = 1;</code>
+ */
+ boolean hasMinimum();
+ /**
+ * <code>optional sint64 minimum = 1;</code>
+ */
+ long getMinimum();
+
+ // optional sint64 maximum = 2;
+ /**
+ * <code>optional sint64 maximum = 2;</code>
+ */
+ boolean hasMaximum();
+ /**
+ * <code>optional sint64 maximum = 2;</code>
+ */
+ long getMaximum();
+
+ // optional sint64 sum = 3;
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ */
+ boolean hasSum();
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ */
+ long getSum();
+ }
+ /**
+ * Protobuf type {@code orc.proto.IntegerStatistics}
+ */
+ public static final class IntegerStatistics extends
+ com.google.protobuf.GeneratedMessage
+ implements IntegerStatisticsOrBuilder {
+ // Use IntegerStatistics.newBuilder() to construct.
+ private IntegerStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private IntegerStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final IntegerStatistics defaultInstance;
+ public static IntegerStatistics getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public IntegerStatistics getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private IntegerStatistics(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 8: {
+ bitField0_ |= 0x00000001;
+ minimum_ = input.readSInt64();
+ break;
+ }
+ case 16: {
+ bitField0_ |= 0x00000002;
+ maximum_ = input.readSInt64();
+ break;
+ }
+ case 24: {
+ bitField0_ |= 0x00000004;
+ sum_ = input.readSInt64();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.orc.OrcProto.IntegerStatistics.class, org.apache.orc.OrcProto.IntegerStatistics.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<IntegerStatistics> PARSER =
+ new com.google.protobuf.AbstractParser<IntegerStatistics>() {
+ public IntegerStatistics parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new IntegerStatistics(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<IntegerStatistics> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // optional sint64 minimum = 1;
+ public static final int MINIMUM_FIELD_NUMBER = 1;
+ private long minimum_;
+ /**
+ * <code>optional sint64 minimum = 1;</code>
+ */
+ public boolean hasMinimum() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional sint64 minimum = 1;</code>
+ */
+ public long getMinimum() {
+ return minimum_;
+ }
+
+ // optional sint64 maximum = 2;
+ public static final int MAXIMUM_FIELD_NUMBER = 2;
+ private long maximum_;
+ /**
+ * <code>optional sint64 maximum = 2;</code>
+ */
+ public boolean hasMaximum() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional sint64 maximum = 2;</code>
+ */
+ public long getMaximum() {
+ return maximum_;
+ }
+
+ // optional sint64 sum = 3;
+ public static final int SUM_FIELD_NUMBER = 3;
+ private long sum_;
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ */
+ public boolean hasSum() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ */
+ public long getSum() {
+ return sum_;
+ }
+
+ private void initFields() {
+ minimum_ = 0L;
+ maximum_ = 0L;
+ sum_ = 0L;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeSInt64(1, minimum_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeSInt64(2, maximum_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeSInt64(3, sum_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeSInt64Size(1, minimum_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeSInt64Size(2, maximum_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeSInt64Size(3, sum_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ public static org.apache.orc.OrcProto.IntegerStatistics parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.orc.OrcProto.IntegerStatistics parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.IntegerStatistics parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.orc.OrcProto.IntegerStatistics parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.IntegerStatistics parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.orc.OrcProto.IntegerStatistics parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.IntegerStatistics parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.orc.OrcProto.IntegerStatistics parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.IntegerStatistics parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.orc.OrcProto.IntegerStatistics parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.orc.OrcProto.IntegerStatistics prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code orc.proto.IntegerStatistics}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.orc.OrcProto.IntegerStatisticsOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.orc.OrcProto.IntegerStatistics.class, org.apache.orc.OrcProto.IntegerStatistics.Builder.class);
+ }
+
+ // Construct using org.apache.orc.OrcProto.IntegerStatistics.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ minimum_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ maximum_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ sum_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_IntegerStatistics_descriptor;
+ }
+
+ public org.apache.orc.OrcProto.IntegerStatistics getDefaultInstanceForType() {
+ return org.apache.orc.OrcProto.IntegerStatistics.getDefaultInstance();
+ }
+
+ public org.apache.orc.OrcProto.IntegerStatistics build() {
+ org.apache.orc.OrcProto.IntegerStatistics result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.orc.OrcProto.IntegerStatistics buildPartial() {
+ org.apache.orc.OrcProto.IntegerStatistics result = new org.apache.orc.OrcProto.IntegerStatistics(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.minimum_ = minimum_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.maximum_ = maximum_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.sum_ = sum_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.orc.OrcProto.IntegerStatistics) {
+ return mergeFrom((org.apache.orc.OrcProto.IntegerStatistics)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.orc.OrcProto.IntegerStatistics other) {
+ if (other == org.apache.orc.OrcProto.IntegerStatistics.getDefaultInstance()) return this;
+ if (other.hasMinimum()) {
+ setMinimum(other.getMinimum());
+ }
+ if (other.hasMaximum()) {
+ setMaximum(other.getMaximum());
+ }
+ if (other.hasSum()) {
+ setSum(other.getSum());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.orc.OrcProto.IntegerStatistics parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.orc.OrcProto.IntegerStatistics) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // optional sint64 minimum = 1;
+ private long minimum_ ;
+ /**
+ * <code>optional sint64 minimum = 1;</code>
+ */
+ public boolean hasMinimum() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional sint64 minimum = 1;</code>
+ */
+ public long getMinimum() {
+ return minimum_;
+ }
+ /**
+ * <code>optional sint64 minimum = 1;</code>
+ */
+ public Builder setMinimum(long value) {
+ bitField0_ |= 0x00000001;
+ minimum_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional sint64 minimum = 1;</code>
+ */
+ public Builder clearMinimum() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ minimum_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // optional sint64 maximum = 2;
+ private long maximum_ ;
+ /**
+ * <code>optional sint64 maximum = 2;</code>
+ */
+ public boolean hasMaximum() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional sint64 maximum = 2;</code>
+ */
+ public long getMaximum() {
+ return maximum_;
+ }
+ /**
+ * <code>optional sint64 maximum = 2;</code>
+ */
+ public Builder setMaximum(long value) {
+ bitField0_ |= 0x00000002;
+ maximum_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional sint64 maximum = 2;</code>
+ */
+ public Builder clearMaximum() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ maximum_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // optional sint64 sum = 3;
+ private long sum_ ;
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ */
+ public boolean hasSum() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ */
+ public long getSum() {
+ return sum_;
+ }
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ */
+ public Builder setSum(long value) {
+ bitField0_ |= 0x00000004;
+ sum_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ */
+ public Builder clearSum() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ sum_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:orc.proto.IntegerStatistics)
+ }
+
+ static {
+ defaultInstance = new IntegerStatistics(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:orc.proto.IntegerStatistics)
+ }
+
+ public interface DoubleStatisticsOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // optional double minimum = 1;
+ /**
+ * <code>optional double minimum = 1;</code>
+ */
+ boolean hasMinimum();
+ /**
+ * <code>optional double minimum = 1;</code>
+ */
+ double getMinimum();
+
+ // optional double maximum = 2;
+ /**
+ * <code>optional double maximum = 2;</code>
+ */
+ boolean hasMaximum();
+ /**
+ * <code>optional double maximum = 2;</code>
+ */
+ double getMaximum();
+
+ // optional double sum = 3;
+ /**
+ * <code>optional double sum = 3;</code>
+ */
+ boolean hasSum();
+ /**
+ * <code>optional double sum = 3;</code>
+ */
+ double getSum();
+ }
+ /**
+ * Protobuf type {@code orc.proto.DoubleStatistics}
+ */
+ public static final class DoubleStatistics extends
+ com.google.protobuf.GeneratedMessage
+ implements DoubleStatisticsOrBuilder {
+ // Use DoubleStatistics.newBuilder() to construct.
+ private DoubleStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private DoubleStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final DoubleStatistics defaultInstance;
+ public static DoubleStatistics getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public DoubleStatistics getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private DoubleStatistics(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 9: {
+ bitField0_ |= 0x00000001;
+ minimum_ = input.readDouble();
+ break;
+ }
+ case 17: {
+ bitField0_ |= 0x00000002;
+ maximum_ = input.readDouble();
+ break;
+ }
+ case 25: {
+ bitField0_ |= 0x00000004;
+ sum_ = input.readDouble();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.orc.OrcProto.DoubleStatistics.class, org.apache.orc.OrcProto.DoubleStatistics.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<DoubleStatistics> PARSER =
+ new com.google.protobuf.AbstractParser<DoubleStatistics>() {
+ public DoubleStatistics parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new DoubleStatistics(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<DoubleStatistics> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // optional double minimum = 1;
+ public static final int MINIMUM_FIELD_NUMBER = 1;
+ private double minimum_;
+ /**
+ * <code>optional double minimum = 1;</code>
+ */
+ public boolean hasMinimum() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional double minimum = 1;</code>
+ */
+ public double getMinimum() {
+ return minimum_;
+ }
+
+ // optional double maximum = 2;
+ public static final int MAXIMUM_FIELD_NUMBER = 2;
+ private double maximum_;
+ /**
+ * <code>optional double maximum = 2;</code>
+ */
+ public boolean hasMaximum() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional double maximum = 2;</code>
+ */
+ public double getMaximum() {
+ return maximum_;
+ }
+
+ // optional double sum = 3;
+ public static final int SUM_FIELD_NUMBER = 3;
+ private double sum_;
+ /**
+ * <code>optional double sum = 3;</code>
+ */
+ public boolean hasSum() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional double sum = 3;</code>
+ */
+ public double getSum() {
+ return sum_;
+ }
+
+ private void initFields() {
+ minimum_ = 0D;
+ maximum_ = 0D;
+ sum_ = 0D;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeDouble(1, minimum_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeDouble(2, maximum_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeDouble(3, sum_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeDoubleSize(1, minimum_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeDoubleSize(2, maximum_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeDoubleSize(3, sum_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ public static org.apache.orc.OrcProto.DoubleStatistics parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.orc.OrcProto.DoubleStatistics parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.DoubleStatistics parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.orc.OrcProto.DoubleStatistics parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.DoubleStatistics parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.orc.OrcProto.DoubleStatistics parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.DoubleStatistics parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.orc.OrcProto.DoubleStatistics parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.DoubleStatistics parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.orc.OrcProto.DoubleStatistics parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.orc.OrcProto.DoubleStatistics prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code orc.proto.DoubleStatistics}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.orc.OrcProto.DoubleStatisticsOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.orc.OrcProto.DoubleStatistics.class, org.apache.orc.OrcProto.DoubleStatistics.Builder.class);
+ }
+
+ // Construct using org.apache.orc.OrcProto.DoubleStatistics.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ minimum_ = 0D;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ maximum_ = 0D;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ sum_ = 0D;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_DoubleStatistics_descriptor;
+ }
+
+ public org.apache.orc.OrcProto.DoubleStatistics getDefaultInstanceForType() {
+ return org.apache.orc.OrcProto.DoubleStatistics.getDefaultInstance();
+ }
+
+ public org.apache.orc.OrcProto.DoubleStatistics build() {
+ org.apache.orc.OrcProto.DoubleStatistics result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.orc.OrcProto.DoubleStatistics buildPartial() {
+ org.apache.orc.OrcProto.DoubleStatistics result = new org.apache.orc.OrcProto.DoubleStatistics(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.minimum_ = minimum_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.maximum_ = maximum_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.sum_ = sum_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.orc.OrcProto.DoubleStatistics) {
+ return mergeFrom((org.apache.orc.OrcProto.DoubleStatistics)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.orc.OrcProto.DoubleStatistics other) {
+ if (other == org.apache.orc.OrcProto.DoubleStatistics.getDefaultInstance()) return this;
+ if (other.hasMinimum()) {
+ setMinimum(other.getMinimum());
+ }
+ if (other.hasMaximum()) {
+ setMaximum(other.getMaximum());
+ }
+ if (other.hasSum()) {
+ setSum(other.getSum());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.orc.OrcProto.DoubleStatistics parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.orc.OrcProto.DoubleStatistics) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // optional double minimum = 1;
+ private double minimum_ ;
+ /**
+ * <code>optional double minimum = 1;</code>
+ */
+ public boolean hasMinimum() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional double minimum = 1;</code>
+ */
+ public double getMinimum() {
+ return minimum_;
+ }
+ /**
+ * <code>optional double minimum = 1;</code>
+ */
+ public Builder setMinimum(double value) {
+ bitField0_ |= 0x00000001;
+ minimum_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional double minimum = 1;</code>
+ */
+ public Builder clearMinimum() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ minimum_ = 0D;
+ onChanged();
+ return this;
+ }
+
+ // optional double maximum = 2;
+ private double maximum_ ;
+ /**
+ * <code>optional double maximum = 2;</code>
+ */
+ public boolean hasMaximum() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional double maximum = 2;</code>
+ */
+ public double getMaximum() {
+ return maximum_;
+ }
+ /**
+ * <code>optional double maximum = 2;</code>
+ */
+ public Builder setMaximum(double value) {
+ bitField0_ |= 0x00000002;
+ maximum_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional double maximum = 2;</code>
+ */
+ public Builder clearMaximum() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ maximum_ = 0D;
+ onChanged();
+ return this;
+ }
+
+ // optional double sum = 3;
+ private double sum_ ;
+ /**
+ * <code>optional double sum = 3;</code>
+ */
+ public boolean hasSum() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional double sum = 3;</code>
+ */
+ public double getSum() {
+ return sum_;
+ }
+ /**
+ * <code>optional double sum = 3;</code>
+ */
+ public Builder setSum(double value) {
+ bitField0_ |= 0x00000004;
+ sum_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional double sum = 3;</code>
+ */
+ public Builder clearSum() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ sum_ = 0D;
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:orc.proto.DoubleStatistics)
+ }
+
+ static {
+ defaultInstance = new DoubleStatistics(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:orc.proto.DoubleStatistics)
+ }
+
+ public interface StringStatisticsOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // optional string minimum = 1;
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ boolean hasMinimum();
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ java.lang.String getMinimum();
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ com.google.protobuf.ByteString
+ getMinimumBytes();
+
+ // optional string maximum = 2;
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ boolean hasMaximum();
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ java.lang.String getMaximum();
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ com.google.protobuf.ByteString
+ getMaximumBytes();
+
+ // optional sint64 sum = 3;
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ *
+ * <pre>
+ * sum will store the total length of all strings in a stripe
+ * </pre>
+ */
+ boolean hasSum();
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ *
+ * <pre>
+ * sum will store the total length of all strings in a stripe
+ * </pre>
+ */
+ long getSum();
+ }
+ /**
+ * Protobuf type {@code orc.proto.StringStatistics}
+ */
+ public static final class StringStatistics extends
+ com.google.protobuf.GeneratedMessage
+ implements StringStatisticsOrBuilder {
+ // Use StringStatistics.newBuilder() to construct.
+ private StringStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private StringStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final StringStatistics defaultInstance;
+ public static StringStatistics getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public StringStatistics getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private StringStatistics(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ minimum_ = input.readBytes();
+ break;
+ }
+ case 18: {
+ bitField0_ |= 0x00000002;
+ maximum_ = input.readBytes();
+ break;
+ }
+ case 24: {
+ bitField0_ |= 0x00000004;
+ sum_ = input.readSInt64();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_StringStatistics_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_StringStatistics_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.orc.OrcProto.StringStatistics.class, org.apache.orc.OrcProto.StringStatistics.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<StringStatistics> PARSER =
+ new com.google.protobuf.AbstractParser<StringStatistics>() {
+ public StringStatistics parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new StringStatistics(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<StringStatistics> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // optional string minimum = 1;
+ public static final int MINIMUM_FIELD_NUMBER = 1;
+ private java.lang.Object minimum_;
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public boolean hasMinimum() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public java.lang.String getMinimum() {
+ java.lang.Object ref = minimum_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ minimum_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getMinimumBytes() {
+ java.lang.Object ref = minimum_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ minimum_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // optional string maximum = 2;
+ public static final int MAXIMUM_FIELD_NUMBER = 2;
+ private java.lang.Object maximum_;
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ public boolean hasMaximum() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ public java.lang.String getMaximum() {
+ java.lang.Object ref = maximum_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ maximum_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ public com.google.protobuf.ByteString
+ getMaximumBytes() {
+ java.lang.Object ref = maximum_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ maximum_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // optional sint64 sum = 3;
+ public static final int SUM_FIELD_NUMBER = 3;
+ private long sum_;
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ *
+ * <pre>
+ * sum will store the total length of all strings in a stripe
+ * </pre>
+ */
+ public boolean hasSum() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ *
+ * <pre>
+ * sum will store the total length of all strings in a stripe
+ * </pre>
+ */
+ public long getSum() {
+ return sum_;
+ }
+
+ private void initFields() {
+ minimum_ = "";
+ maximum_ = "";
+ sum_ = 0L;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getMinimumBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeBytes(2, getMaximumBytes());
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeSInt64(3, sum_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getMinimumBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(2, getMaximumBytes());
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeSInt64Size(3, sum_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ public static org.apache.orc.OrcProto.StringStatistics parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.orc.OrcProto.StringStatistics parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.StringStatistics parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.orc.OrcProto.StringStatistics parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.StringStatistics parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.orc.OrcProto.StringStatistics parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.StringStatistics parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.orc.OrcProto.StringStatistics parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.StringStatistics parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.orc.OrcProto.StringStatistics parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.orc.OrcProto.StringStatistics prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code orc.proto.StringStatistics}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.orc.OrcProto.StringStatisticsOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_StringStatistics_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_StringStatistics_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.orc.OrcProto.StringStatistics.class, org.apache.orc.OrcProto.StringStatistics.Builder.class);
+ }
+
+ // Construct using org.apache.orc.OrcProto.StringStatistics.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ minimum_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ maximum_ = "";
+ bitField0_ = (bitField0_ & ~0x00000002);
+ sum_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_StringStatistics_descriptor;
+ }
+
+ public org.apache.orc.OrcProto.StringStatistics getDefaultInstanceForType() {
+ return org.apache.orc.OrcProto.StringStatistics.getDefaultInstance();
+ }
+
+ public org.apache.orc.OrcProto.StringStatistics build() {
+ org.apache.orc.OrcProto.StringStatistics result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.orc.OrcProto.StringStatistics buildPartial() {
+ org.apache.orc.OrcProto.StringStatistics result = new org.apache.orc.OrcProto.StringStatistics(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.minimum_ = minimum_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.maximum_ = maximum_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.sum_ = sum_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.orc.OrcProto.StringStatistics) {
+ return mergeFrom((org.apache.orc.OrcProto.StringStatistics)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.orc.OrcProto.StringStatistics other) {
+ if (other == org.apache.orc.OrcProto.StringStatistics.getDefaultInstance()) return this;
+ if (other.hasMinimum()) {
+ bitField0_ |= 0x00000001;
+ minimum_ = other.minimum_;
+ onChanged();
+ }
+ if (other.hasMaximum()) {
+ bitField0_ |= 0x00000002;
+ maximum_ = other.maximum_;
+ onChanged();
+ }
+ if (other.hasSum()) {
+ setSum(other.getSum());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.orc.OrcProto.StringStatistics parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.orc.OrcProto.StringStatistics) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // optional string minimum = 1;
+ private java.lang.Object minimum_ = "";
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public boolean hasMinimum() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public java.lang.String getMinimum() {
+ java.lang.Object ref = minimum_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ minimum_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getMinimumBytes() {
+ java.lang.Object ref = minimum_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ minimum_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public Builder setMinimum(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ minimum_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public Builder clearMinimum() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ minimum_ = getDefaultInstance().getMinimum();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public Builder setMinimumBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ minimum_ = value;
+ onChanged();
+ return this;
+ }
+
+ // optional string maximum = 2;
+ private java.lang.Object maximum_ = "";
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ public boolean hasMaximum() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ public java.lang.String getMaximum() {
+ java.lang.Object ref = maximum_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ maximum_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ public com.google.protobuf.ByteString
+ getMaximumBytes() {
+ java.lang.Object ref = maximum_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ maximum_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ public Builder setMaximum(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000002;
+ maximum_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ public Builder clearMaximum() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ maximum_ = getDefaultInstance().getMaximum();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ public Builder setMaximumBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000002;
+ maximum_ = value;
+ onChanged();
+ return this;
+ }
+
+ // optional sint64 sum = 3;
+ private long sum_ ;
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ *
+ * <pre>
+ * sum will store the total length of all strings in a stripe
+ * </pre>
+ */
+ public boolean hasSum() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ *
+ * <pre>
+ * sum will store the total length of all strings in a stripe
+ * </pre>
+ */
+ public long getSum() {
+ return sum_;
+ }
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ *
+ * <pre>
+ * sum will store the total length of all strings in a stripe
+ * </pre>
+ */
+ public Builder setSum(long value) {
+ bitField0_ |= 0x00000004;
+ sum_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional sint64 sum = 3;</code>
+ *
+ * <pre>
+ * sum will store the total length of all strings in a stripe
+ * </pre>
+ */
+ public Builder clearSum() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ sum_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:orc.proto.StringStatistics)
+ }
+
+ static {
+ defaultInstance = new StringStatistics(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:orc.proto.StringStatistics)
+ }
+
+ public interface BucketStatisticsOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // repeated uint64 count = 1 [packed = true];
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ java.util.List<java.lang.Long> getCountList();
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ int getCountCount();
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ long getCount(int index);
+ }
+ /**
+ * Protobuf type {@code orc.proto.BucketStatistics}
+ */
+ public static final class BucketStatistics extends
+ com.google.protobuf.GeneratedMessage
+ implements BucketStatisticsOrBuilder {
+ // Use BucketStatistics.newBuilder() to construct.
+ private BucketStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private BucketStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final BucketStatistics defaultInstance;
+ public static BucketStatistics getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public BucketStatistics getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private BucketStatistics(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 8: {
+ if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
+ count_ = new java.util.ArrayList<java.lang.Long>();
+ mutable_bitField0_ |= 0x00000001;
+ }
+ count_.add(input.readUInt64());
+ break;
+ }
+ case 10: {
+ int length = input.readRawVarint32();
+ int limit = input.pushLimit(length);
+ if (!((mutable_bitField0_ & 0x00000001) == 0x00000001) && input.getBytesUntilLimit() > 0) {
+ count_ = new java.util.ArrayList<java.lang.Long>();
+ mutable_bitField0_ |= 0x00000001;
+ }
+ while (input.getBytesUntilLimit() > 0) {
+ count_.add(input.readUInt64());
+ }
+ input.popLimit(limit);
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
+ count_ = java.util.Collections.unmodifiableList(count_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_BucketStatistics_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_BucketStatistics_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.orc.OrcProto.BucketStatistics.class, org.apache.orc.OrcProto.BucketStatistics.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<BucketStatistics> PARSER =
+ new com.google.protobuf.AbstractParser<BucketStatistics>() {
+ public BucketStatistics parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new BucketStatistics(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<BucketStatistics> getParserForType() {
+ return PARSER;
+ }
+
+ // repeated uint64 count = 1 [packed = true];
+ public static final int COUNT_FIELD_NUMBER = 1;
+ private java.util.List<java.lang.Long> count_;
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public java.util.List<java.lang.Long>
+ getCountList() {
+ return count_;
+ }
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public int getCountCount() {
+ return count_.size();
+ }
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public long getCount(int index) {
+ return count_.get(index);
+ }
+ private int countMemoizedSerializedSize = -1;
+
+ private void initFields() {
+ count_ = java.util.Collections.emptyList();
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (getCountList().size() > 0) {
+ output.writeRawVarint32(10);
+ output.writeRawVarint32(countMemoizedSerializedSize);
+ }
+ for (int i = 0; i < count_.size(); i++) {
+ output.writeUInt64NoTag(count_.get(i));
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ {
+ int dataSize = 0;
+ for (int i = 0; i < count_.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream
+ .computeUInt64SizeNoTag(count_.get(i));
+ }
+ size += dataSize;
+ if (!getCountList().isEmpty()) {
+ size += 1;
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32SizeNoTag(dataSize);
+ }
+ countMemoizedSerializedSize = dataSize;
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ public static org.apache.orc.OrcProto.BucketStatistics parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.orc.OrcProto.BucketStatistics parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.BucketStatistics parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.orc.OrcProto.BucketStatistics parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.BucketStatistics parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.orc.OrcProto.BucketStatistics parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.BucketStatistics parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.orc.OrcProto.BucketStatistics parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.orc.OrcProto.BucketStatistics parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.orc.OrcProto.BucketStatistics parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.orc.OrcProto.BucketStatistics prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code orc.proto.BucketStatistics}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.orc.OrcProto.BucketStatisticsOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_BucketStatistics_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_BucketStatistics_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.orc.OrcProto.BucketStatistics.class, org.apache.orc.OrcProto.BucketStatistics.Builder.class);
+ }
+
+ // Construct using org.apache.orc.OrcProto.BucketStatistics.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ count_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000001);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_BucketStatistics_descriptor;
+ }
+
+ public org.apache.orc.OrcProto.BucketStatistics getDefaultInstanceForType() {
+ return org.apache.orc.OrcProto.BucketStatistics.getDefaultInstance();
+ }
+
+ public org.apache.orc.OrcProto.BucketStatistics build() {
+ org.apache.orc.OrcProto.BucketStatistics result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.orc.OrcProto.BucketStatistics buildPartial() {
+ org.apache.orc.OrcProto.BucketStatistics result = new org.apache.orc.OrcProto.BucketStatistics(this);
+ int from_bitField0_ = bitField0_;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ count_ = java.util.Collections.unmodifiableList(count_);
+ bitField0_ = (bitField0_ & ~0x00000001);
+ }
+ result.count_ = count_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.orc.OrcProto.BucketStatistics) {
+ return mergeFrom((org.apache.orc.OrcProto.BucketStatistics)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.orc.OrcProto.BucketStatistics other) {
+ if (other == org.apache.orc.OrcProto.BucketStatistics.getDefaultInstance()) return this;
+ if (!other.count_.isEmpty()) {
+ if (count_.isEmpty()) {
+ count_ = other.count_;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ } else {
+ ensureCountIsMutable();
+ count_.addAll(other.count_);
+ }
+ onChanged();
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.orc.OrcProto.BucketStatistics parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.orc.OrcProto.BucketStatistics) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // repeated uint64 count = 1 [packed = true];
+ private java.util.List<java.lang.Long> count_ = java.util.Collections.emptyList();
+ private void ensureCountIsMutable() {
+ if (!((bitField0_ & 0x00000001) == 0x00000001)) {
+ count_ = new java.util.ArrayList<java.lang.Long>(count_);
+ bitField0_ |= 0x00000001;
+ }
+ }
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public java.util.List<java.lang.Long>
+ getCountList() {
+ return java.util.Collections.unmodifiableList(count_);
+ }
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public int getCountCount() {
+ return count_.size();
+ }
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public long getCount(int index) {
+ return count_.get(index);
+ }
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public Builder setCount(
+ int index, long value) {
+ ensureCountIsMutable();
+ count_.set(index, value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public Builder addCount(long value) {
+ ensureCountIsMutable();
+ count_.add(value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public Builder addAllCount(
+ java.lang.Iterable<? extends java.lang.Long> values) {
+ ensureCountIsMutable();
+ super.addAll(values, count_);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated uint64 count = 1 [packed = true];</code>
+ */
+ public Builder clearCount() {
+ count_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000001);
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:orc.proto.BucketStatistics)
+ }
+
+ static {
+ defaultInstance = new BucketStatistics(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:orc.proto.BucketStatistics)
+ }
+
+ public interface DecimalStatisticsOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // optional string minimum = 1;
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ boolean hasMinimum();
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ java.lang.String getMinimum();
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ com.google.protobuf.ByteString
+ getMinimumBytes();
+
+ // optional string maximum = 2;
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ boolean hasMaximum();
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ java.lang.String getMaximum();
+ /**
+ * <code>optional string maximum = 2;</code>
+ */
+ com.google.protobuf.ByteString
+ getMaximumBytes();
+
+ // optional string sum = 3;
+ /**
+ * <code>optional string sum = 3;</code>
+ */
+ boolean hasSum();
+ /**
+ * <code>optional string sum = 3;</code>
+ */
+ java.lang.String getSum();
+ /**
+ * <code>optional string sum = 3;</code>
+ */
+ com.google.protobuf.ByteString
+ getSumBytes();
+ }
+ /**
+ * Protobuf type {@code orc.proto.DecimalStatistics}
+ */
+ public static final class DecimalStatistics extends
+ com.google.protobuf.GeneratedMessage
+ implements DecimalStatisticsOrBuilder {
+ // Use DecimalStatistics.newBuilder() to construct.
+ private DecimalStatistics(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private DecimalStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final DecimalStatistics defaultInstance;
+ public static DecimalStatistics getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public DecimalStatistics getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private DecimalStatistics(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ minimum_ = input.readBytes();
+ break;
+ }
+ case 18: {
+ bitField0_ |= 0x00000002;
+ maximum_ = input.readBytes();
+ break;
+ }
+ case 26: {
+ bitField0_ |= 0x00000004;
+ sum_ = input.readBytes();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_DecimalStatistics_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.orc.OrcProto.internal_static_orc_proto_DecimalStatistics_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.orc.OrcProto.DecimalStatistics.class, org.apache.orc.OrcProto.DecimalStatistics.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<DecimalStatistics> PARSER =
+ new com.google.protobuf.AbstractParser<DecimalStatistics>() {
+ public DecimalStatistics parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new DecimalStatistics(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<DecimalStatistics> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // optional string minimum = 1;
+ public static final int MINIMUM_FIELD_NUMBER = 1;
+ private java.lang.Object minimum_;
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public boolean hasMinimum() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public java.lang.String getMinimum() {
+ java.lang.Object ref = minimum_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ minimum_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string minimum = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getMinimumBytes() {
+ java.lang.Object ref = minimum_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ minimum_ = b;
+
<TRUNCATED>
[14/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/BinaryColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/BinaryColumnStatistics.java b/orc/src/java/org/apache/orc/BinaryColumnStatistics.java
new file mode 100644
index 0000000..19db98a
--- /dev/null
+++ b/orc/src/java/org/apache/orc/BinaryColumnStatistics.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import org.apache.orc.ColumnStatistics;
+
+/**
+ * Statistics for binary columns.
+ */
+public interface BinaryColumnStatistics extends ColumnStatistics {
+ long getSum();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/BooleanColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/BooleanColumnStatistics.java b/orc/src/java/org/apache/orc/BooleanColumnStatistics.java
new file mode 100644
index 0000000..af08f06
--- /dev/null
+++ b/orc/src/java/org/apache/orc/BooleanColumnStatistics.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import org.apache.orc.ColumnStatistics;
+
+/**
+ * Statistics for boolean columns.
+ */
+public interface BooleanColumnStatistics extends ColumnStatistics {
+ long getFalseCount();
+
+ long getTrueCount();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/ColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/ColumnStatistics.java b/orc/src/java/org/apache/orc/ColumnStatistics.java
new file mode 100644
index 0000000..72d8fbf
--- /dev/null
+++ b/orc/src/java/org/apache/orc/ColumnStatistics.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+/**
+ * Statistics that are available for all types of columns.
+ */
+public interface ColumnStatistics {
+ /**
+ * Get the number of values in this column. It will differ from the number
+ * of rows because of NULL values and repeated values.
+ * @return the number of values
+ */
+ long getNumberOfValues();
+
+ /**
+ * Returns true if there are nulls in the scope of column statistics.
+ * @return true if null present else false
+ */
+ boolean hasNull();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/CompressionCodec.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/CompressionCodec.java b/orc/src/java/org/apache/orc/CompressionCodec.java
new file mode 100644
index 0000000..3421969
--- /dev/null
+++ b/orc/src/java/org/apache/orc/CompressionCodec.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.EnumSet;
+
+import javax.annotation.Nullable;
+
+public interface CompressionCodec {
+
+ enum Modifier {
+ /* speed/compression tradeoffs */
+ FASTEST,
+ FAST,
+ DEFAULT,
+ /* data sensitivity modifiers */
+ TEXT,
+ BINARY
+ };
+
+ /**
+ * Compress the in buffer to the out buffer.
+ * @param in the bytes to compress
+ * @param out the uncompressed bytes
+ * @param overflow put any additional bytes here
+ * @return true if the output is smaller than input
+ * @throws IOException
+ */
+ boolean compress(ByteBuffer in, ByteBuffer out, ByteBuffer overflow
+ ) throws IOException;
+
+ /**
+ * Decompress the in buffer to the out buffer.
+ * @param in the bytes to decompress
+ * @param out the decompressed bytes
+ * @throws IOException
+ */
+ void decompress(ByteBuffer in, ByteBuffer out) throws IOException;
+
+ /**
+ * Produce a modified compression codec if the underlying algorithm allows
+ * modification.
+ *
+ * This does not modify the current object, but returns a new object if
+ * modifications are possible. Returns the same object if no modifications
+ * are possible.
+ * @param modifiers compression modifiers
+ * @return codec for use after optional modification
+ */
+ CompressionCodec modify(@Nullable EnumSet<Modifier> modifiers);
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/CompressionKind.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/CompressionKind.java b/orc/src/java/org/apache/orc/CompressionKind.java
new file mode 100644
index 0000000..f684bef
--- /dev/null
+++ b/orc/src/java/org/apache/orc/CompressionKind.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+/**
+ * An enumeration that lists the generic compression algorithms that
+ * can be applied to ORC files.
+ */
+public enum CompressionKind {
+ NONE, ZLIB, SNAPPY, LZO
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/DataReader.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/DataReader.java b/orc/src/java/org/apache/orc/DataReader.java
new file mode 100644
index 0000000..3a5f854
--- /dev/null
+++ b/orc/src/java/org/apache/orc/DataReader.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+
+/** An abstract data reader that IO formats can use to read bytes from underlying storage. */
+public interface DataReader {
+
+ /** Opens the DataReader, making it ready to use. */
+ void open() throws IOException;
+
+ /** Closes the DataReader. */
+ void close() throws IOException;
+
+ /** Reads the data.
+ *
+ * Note that for the cases such as zero-copy read, caller must release the disk ranges
+ * produced after being done with them. Call isTrackingDiskRanges to find out if this is needed.
+ * @param range List if disk ranges to read. Ranges with data will be ignored.
+ * @param baseOffset Base offset from the start of the file of the ranges in disk range list.
+ * @param doForceDirect Whether the data should be read into direct buffers.
+ * @return New or modified list of DiskRange-s, where all the ranges are filled with data.
+ */
+ DiskRangeList readFileData(
+ DiskRangeList range, long baseOffset, boolean doForceDirect) throws IOException;
+
+
+ /**
+ * Whether the user should release buffers created by readFileData. See readFileData javadoc.
+ */
+ boolean isTrackingDiskRanges();
+
+ /**
+ * Releases buffers created by readFileData. See readFileData javadoc.
+ * @param toRelease The buffer to release.
+ */
+ void releaseBuffer(ByteBuffer toRelease);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/DateColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/DateColumnStatistics.java b/orc/src/java/org/apache/orc/DateColumnStatistics.java
new file mode 100644
index 0000000..cdd01af
--- /dev/null
+++ b/orc/src/java/org/apache/orc/DateColumnStatistics.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import org.apache.orc.ColumnStatistics;
+
+import java.util.Date;
+
+/**
+ * Statistics for DATE columns.
+ */
+public interface DateColumnStatistics extends ColumnStatistics {
+ /**
+ * Get the minimum value for the column.
+ * @return minimum value
+ */
+ Date getMinimum();
+
+ /**
+ * Get the maximum value for the column.
+ * @return maximum value
+ */
+ Date getMaximum();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/DecimalColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/DecimalColumnStatistics.java b/orc/src/java/org/apache/orc/DecimalColumnStatistics.java
new file mode 100644
index 0000000..51b6d7d
--- /dev/null
+++ b/orc/src/java/org/apache/orc/DecimalColumnStatistics.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.orc.ColumnStatistics;
+
+/**
+ * Statistics for decimal columns.
+ */
+public interface DecimalColumnStatistics extends ColumnStatistics {
+
+ /**
+ * Get the minimum value for the column.
+ * @return the minimum value
+ */
+ HiveDecimal getMinimum();
+
+ /**
+ * Get the maximum value for the column.
+ * @return the maximum value
+ */
+ HiveDecimal getMaximum();
+
+ /**
+ * Get the sum of the values of the column.
+ * @return the sum
+ */
+ HiveDecimal getSum();
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/DoubleColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/DoubleColumnStatistics.java b/orc/src/java/org/apache/orc/DoubleColumnStatistics.java
new file mode 100644
index 0000000..00c728f
--- /dev/null
+++ b/orc/src/java/org/apache/orc/DoubleColumnStatistics.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import org.apache.orc.ColumnStatistics;
+
+/**
+ * Statistics for float and double columns.
+ */
+public interface DoubleColumnStatistics extends ColumnStatistics {
+
+ /**
+ * Get the smallest value in the column. Only defined if getNumberOfValues
+ * is non-zero.
+ * @return the minimum
+ */
+ double getMinimum();
+
+ /**
+ * Get the largest value in the column. Only defined if getNumberOfValues
+ * is non-zero.
+ * @return the maximum
+ */
+ double getMaximum();
+
+ /**
+ * Get the sum of the values in the column.
+ * @return the sum
+ */
+ double getSum();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/FileMetaInfo.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/FileMetaInfo.java b/orc/src/java/org/apache/orc/FileMetaInfo.java
new file mode 100644
index 0000000..d3cac3b
--- /dev/null
+++ b/orc/src/java/org/apache/orc/FileMetaInfo.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/**
+ * FileMetaInfo - represents file metadata stored in footer and postscript sections of the file
+ * that is useful for Reader implementation
+ *
+ */
+public class FileMetaInfo {
+ public ByteBuffer footerMetaAndPsBuffer;
+ public final String compressionType;
+ public final int bufferSize;
+ public final int metadataSize;
+ public final ByteBuffer footerBuffer;
+ public final List<Integer> versionList;
+ public final OrcFile.WriterVersion writerVersion;
+
+
+ /** Ctor used when reading splits - no version list or full footer buffer. */
+ public FileMetaInfo(String compressionType, int bufferSize, int metadataSize,
+ ByteBuffer footerBuffer, OrcFile.WriterVersion writerVersion) {
+ this(compressionType, bufferSize, metadataSize, footerBuffer, null,
+ writerVersion, null);
+ }
+
+ /** Ctor used when creating file info during init and when getting a new one. */
+ public FileMetaInfo(String compressionType, int bufferSize, int metadataSize,
+ ByteBuffer footerBuffer, List<Integer> versionList,
+ OrcFile.WriterVersion writerVersion,
+ ByteBuffer fullFooterBuffer) {
+ this.compressionType = compressionType;
+ this.bufferSize = bufferSize;
+ this.metadataSize = metadataSize;
+ this.footerBuffer = footerBuffer;
+ this.versionList = versionList;
+ this.writerVersion = writerVersion;
+ this.footerMetaAndPsBuffer = fullFooterBuffer;
+ }
+
+ public OrcFile.WriterVersion getWriterVersion() {
+ return writerVersion;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/FileMetadata.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/FileMetadata.java b/orc/src/java/org/apache/orc/FileMetadata.java
new file mode 100644
index 0000000..d63bdcc
--- /dev/null
+++ b/orc/src/java/org/apache/orc/FileMetadata.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import java.util.List;
+
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcProto;
+import org.apache.orc.StripeInformation;
+
+/**
+ * Cached file metadata. Right now, it caches everything; we don't have to store all the
+ * protobuf structs actually, we could just store what we need, but that would require that
+ * ORC stop depending on them too. Luckily, they shouldn't be very big.
+ */
+public interface FileMetadata {
+ boolean isOriginalFormat();
+
+ List<StripeInformation> getStripes();
+
+ CompressionKind getCompressionKind();
+
+ int getCompressionBufferSize();
+
+ int getRowIndexStride();
+
+ int getColumnCount();
+
+ int getFlattenedColumnCount();
+
+ long getFileId();
+
+ List<Integer> getVersionList();
+
+ int getMetadataSize();
+
+ int getWriterVersionNum();
+
+ List<OrcProto.Type> getTypes();
+
+ List<OrcProto.StripeStatistics> getStripeStats();
+
+ long getContentLength();
+
+ long getNumberOfRows();
+
+ List<OrcProto.ColumnStatistics> getFileStats();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/IntegerColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/IntegerColumnStatistics.java b/orc/src/java/org/apache/orc/IntegerColumnStatistics.java
new file mode 100644
index 0000000..1a162ff
--- /dev/null
+++ b/orc/src/java/org/apache/orc/IntegerColumnStatistics.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import org.apache.orc.ColumnStatistics;
+
+/**
+ * Statistics for all of the integer columns, such as byte, short, int, and
+ * long.
+ */
+public interface IntegerColumnStatistics extends ColumnStatistics {
+ /**
+ * Get the smallest value in the column. Only defined if getNumberOfValues
+ * is non-zero.
+ * @return the minimum
+ */
+ long getMinimum();
+
+ /**
+ * Get the largest value in the column. Only defined if getNumberOfValues
+ * is non-zero.
+ * @return the maximum
+ */
+ long getMaximum();
+
+ /**
+ * Is the sum defined? If the sum overflowed the counter this will be false.
+ * @return is the sum available
+ */
+ boolean isSumDefined();
+
+ /**
+ * Get the sum of the column. Only valid if isSumDefined returns true.
+ * @return the sum of the column
+ */
+ long getSum();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/OrcConf.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/OrcConf.java b/orc/src/java/org/apache/orc/OrcConf.java
new file mode 100644
index 0000000..6fcbb72
--- /dev/null
+++ b/orc/src/java/org/apache/orc/OrcConf.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.Properties;
+
+/**
+ * Define the configuration properties that Orc understands.
+ */
+public enum OrcConf {
+ STRIPE_SIZE("orc.stripe.size", "hive.exec.orc.default.stripe.size",
+ 64L * 1024 * 1024,
+ "Define the default ORC stripe size, in bytes."),
+ BLOCK_SIZE("orc.block.size", "hive.exec.orc.default.block.size",
+ 256L * 1024 * 1024,
+ "Define the default file system block size for ORC files."),
+ ENABLE_INDEXES("orc.create.index", "orc.create.index", true,
+ "Should the ORC writer create indexes as part of the file."),
+ ROW_INDEX_STRIDE("orc.row.index.stride",
+ "hive.exec.orc.default.row.index.stride", 10000,
+ "Define the default ORC index stride in number of rows. (Stride is the\n"+
+ " number of rows n index entry represents.)"),
+ BUFFER_SIZE("orc.compress.size", "hive.exec.orc.default.buffer.size",
+ 256 * 1024, "Define the default ORC buffer size, in bytes."),
+ BLOCK_PADDING("orc.block.padding", "hive.exec.orc.default.block.padding",
+ true,
+ "Define whether stripes should be padded to the HDFS block boundaries."),
+ COMPRESS("orc.compress", "hive.exec.orc.default.compress", "ZLIB",
+ "Define the default compression codec for ORC file"),
+ WRITE_FORMAT("orc.write.format", "hive.exec.orc.write.format", "0.12",
+ "Define the version of the file to write. Possible values are 0.11 and\n"+
+ " 0.12. If this parameter is not defined, ORC will use the run\n" +
+ " length encoding (RLE) introduced in Hive 0.12."),
+ ENCODING_STRATEGY("orc.encoding.strategy", "hive.exec.orc.encoding.strategy",
+ "SPEED",
+ "Define the encoding strategy to use while writing data. Changing this\n"+
+ "will only affect the light weight encoding for integers. This\n" +
+ "flag will not change the compression level of higher level\n" +
+ "compression codec (like ZLIB)."),
+ COMPRESSION_STRATEGY("orc.compression.strategy",
+ "hive.exec.orc.compression.strategy", "SPEED",
+ "Define the compression strategy to use while writing data.\n" +
+ "This changes the compression level of higher level compression\n" +
+ "codec (like ZLIB)."),
+ BLOCK_PADDING_TOLERANCE("orc.block.padding.tolerance",
+ "hive.exec.orc.block.padding.tolerance", 0.05,
+ "Define the tolerance for block padding as a decimal fraction of\n" +
+ "stripe size (for example, the default value 0.05 is 5% of the\n" +
+ "stripe size). For the defaults of 64Mb ORC stripe and 256Mb HDFS\n" +
+ "blocks, the default block padding tolerance of 5% will\n" +
+ "reserve a maximum of 3.2Mb for padding within the 256Mb block.\n" +
+ "In that case, if the available size within the block is more than\n"+
+ "3.2Mb, a new smaller stripe will be inserted to fit within that\n" +
+ "space. This will make sure that no stripe written will block\n" +
+ " boundaries and cause remote reads within a node local task."),
+ BLOOM_FILTER_FPP("orc.bloom.filter.fpp", "orc.default.bloom.fpp", 0.05,
+ "Define the default false positive probability for bloom filters."),
+ USE_ZEROCOPY("orc.use.zerocopy", "hive.exec.orc.zerocopy", false,
+ "Use zerocopy reads with ORC. (This requires Hadoop 2.3 or later.)"),
+ SKIP_CORRUPT_DATA("orc.skip.corrupt.data", "hive.exec.orc.skip.corrupt.data",
+ false,
+ "If ORC reader encounters corrupt data, this value will be used to\n" +
+ "determine whether to skip the corrupt data or throw exception.\n" +
+ "The default behavior is to throw exception."),
+ MEMORY_POOL("orc.memory.pool", "hive.exec.orc.memory.pool", 0.5,
+ "Maximum fraction of heap that can be used by ORC file writers"),
+ DICTIONARY_KEY_SIZE_THRESHOLD("orc.dictionary.key.threshold",
+ "hive.exec.orc.dictionary.key.size.threshold",
+ 0.8,
+ "If the number of distinct keys in a dictionary is greater than this\n" +
+ "fraction of the total number of non-null rows, turn off \n" +
+ "dictionary encoding. Use 1 to always use dictionary encoding."),
+ ROW_INDEX_STRIDE_DICTIONARY_CHECK("orc.dictionary.early.check",
+ "hive.orc.row.index.stride.dictionary.check",
+ true,
+ "If enabled dictionary check will happen after first row index stride\n" +
+ "(default 10000 rows) else dictionary check will happen before\n" +
+ "writing first stripe. In both cases, the decision to use\n" +
+ "dictionary or not will be retained thereafter."),
+ BLOOM_FILTER_COLUMNS("orc.bloom.filter.columns", "orc.bloom.filter.columns",
+ "", "List of columns to create bloom filters for when writing.")
+ ;
+
+ private final String attribute;
+ private final String hiveConfName;
+ private final Object defaultValue;
+ private final String description;
+
+ OrcConf(String attribute,
+ String hiveConfName,
+ Object defaultValue,
+ String description) {
+ this.attribute = attribute;
+ this.hiveConfName = hiveConfName;
+ this.defaultValue = defaultValue;
+ this.description = description;
+ }
+
+ public String getAttribute() {
+ return attribute;
+ }
+
+ public String getHiveConfName() {
+ return hiveConfName;
+ }
+
+ public Object getDefaultValue() {
+ return defaultValue;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ private String lookupValue(Properties tbl, Configuration conf) {
+ String result = null;
+ if (tbl != null) {
+ result = tbl.getProperty(attribute);
+ }
+ if (result == null && conf != null) {
+ result = conf.get(attribute);
+ if (result == null) {
+ result = conf.get(hiveConfName);
+ }
+ }
+ return result;
+ }
+
+ public long getLong(Properties tbl, Configuration conf) {
+ String value = lookupValue(tbl, conf);
+ if (value != null) {
+ return Long.parseLong(value);
+ }
+ return ((Number) defaultValue).longValue();
+ }
+
+ public long getLong(Configuration conf) {
+ return getLong(null, conf);
+ }
+
+ public String getString(Properties tbl, Configuration conf) {
+ String value = lookupValue(tbl, conf);
+ return value == null ? (String) defaultValue : value;
+ }
+
+ public String getString(Configuration conf) {
+ return getString(null, conf);
+ }
+
+ public boolean getBoolean(Properties tbl, Configuration conf) {
+ String value = lookupValue(tbl, conf);
+ if (value != null) {
+ return Boolean.parseBoolean(value);
+ }
+ return (Boolean) defaultValue;
+ }
+
+ public boolean getBoolean(Configuration conf) {
+ return getBoolean(null, conf);
+ }
+
+ public double getDouble(Properties tbl, Configuration conf) {
+ String value = lookupValue(tbl, conf);
+ if (value != null) {
+ return Double.parseDouble(value);
+ }
+ return ((Number) defaultValue).doubleValue();
+ }
+
+ public double getDouble(Configuration conf) {
+ return getDouble(null, conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/OrcFile.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/OrcFile.java b/orc/src/java/org/apache/orc/OrcFile.java
new file mode 100644
index 0000000..9ea0b52
--- /dev/null
+++ b/orc/src/java/org/apache/orc/OrcFile.java
@@ -0,0 +1,514 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.orc.impl.MemoryManager;
+
+/**
+ * Contains factory methods to read or write ORC files.
+ */
+public class OrcFile {
+ public static final String MAGIC = "ORC";
+
+ /**
+ * Create a version number for the ORC file format, so that we can add
+ * non-forward compatible changes in the future. To make it easier for users
+ * to understand the version numbers, we use the Hive release number that
+ * first wrote that version of ORC files.
+ *
+ * Thus, if you add new encodings or other non-forward compatible changes
+ * to ORC files, which prevent the old reader from reading the new format,
+ * you should change these variable to reflect the next Hive release number.
+ * Non-forward compatible changes should never be added in patch releases.
+ *
+ * Do not make any changes that break backwards compatibility, which would
+ * prevent the new reader from reading ORC files generated by any released
+ * version of Hive.
+ */
+ public enum Version {
+ V_0_11("0.11", 0, 11),
+ V_0_12("0.12", 0, 12);
+
+ public static final Version CURRENT = V_0_12;
+
+ private final String name;
+ private final int major;
+ private final int minor;
+
+ Version(String name, int major, int minor) {
+ this.name = name;
+ this.major = major;
+ this.minor = minor;
+ }
+
+ public static Version byName(String name) {
+ for(Version version: values()) {
+ if (version.name.equals(name)) {
+ return version;
+ }
+ }
+ throw new IllegalArgumentException("Unknown ORC version " + name);
+ }
+
+ /**
+ * Get the human readable name for the version.
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Get the major version number.
+ */
+ public int getMajor() {
+ return major;
+ }
+
+ /**
+ * Get the minor version number.
+ */
+ public int getMinor() {
+ return minor;
+ }
+ }
+
+ /**
+ * Records the version of the writer in terms of which bugs have been fixed.
+ * For bugs in the writer, but the old readers already read the new data
+ * correctly, bump this version instead of the Version.
+ */
+ public enum WriterVersion {
+ ORIGINAL(0),
+ HIVE_8732(1), // corrupted stripe/file maximum column statistics
+ HIVE_4243(2), // use real column names from Hive tables
+ // Don't use any magic numbers here except for the below:
+ FUTURE(Integer.MAX_VALUE); // a version from a future writer
+
+ private final int id;
+
+ public int getId() {
+ return id;
+ }
+
+ WriterVersion(int id) {
+ this.id = id;
+ }
+
+ private static final WriterVersion[] values;
+ static {
+ // Assumes few non-negative values close to zero.
+ int max = Integer.MIN_VALUE;
+ for (WriterVersion v : WriterVersion.values()) {
+ if (v.id < 0) throw new AssertionError();
+ if (v.id > max && FUTURE.id != v.id) {
+ max = v.id;
+ }
+ }
+ values = new WriterVersion[max + 1];
+ for (WriterVersion v : WriterVersion.values()) {
+ if (v.id < values.length) {
+ values[v.id] = v;
+ }
+ }
+ }
+
+ public static WriterVersion from(int val) {
+ if (val == FUTURE.id) return FUTURE; // Special handling for the magic value.
+ return values[val];
+ }
+ }
+
+ public enum EncodingStrategy {
+ SPEED, COMPRESSION
+ }
+
+ public enum CompressionStrategy {
+ SPEED, COMPRESSION
+ }
+
+ // unused
+ protected OrcFile() {}
+
+ public static class ReaderOptions {
+ private final Configuration conf;
+ private FileSystem filesystem;
+ private FileMetaInfo fileMetaInfo; // TODO: this comes from some place.
+ private long maxLength = Long.MAX_VALUE;
+ private FileMetadata fullFileMetadata; // Propagate from LLAP cache.
+
+ public ReaderOptions(Configuration conf) {
+ this.conf = conf;
+ }
+
+ public ReaderOptions fileMetaInfo(FileMetaInfo info) {
+ fileMetaInfo = info;
+ return this;
+ }
+
+ public ReaderOptions filesystem(FileSystem fs) {
+ this.filesystem = fs;
+ return this;
+ }
+
+ public ReaderOptions maxLength(long val) {
+ maxLength = val;
+ return this;
+ }
+
+ public ReaderOptions fileMetadata(FileMetadata metadata) {
+ this.fullFileMetadata = metadata;
+ return this;
+ }
+
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ public FileSystem getFilesystem() {
+ return filesystem;
+ }
+
+ public FileMetaInfo getFileMetaInfo() {
+ return fileMetaInfo;
+ }
+
+ public long getMaxLength() {
+ return maxLength;
+ }
+
+ public FileMetadata getFileMetadata() {
+ return fullFileMetadata;
+ }
+ }
+
+ public static ReaderOptions readerOptions(Configuration conf) {
+ return new ReaderOptions(conf);
+ }
+
+ public interface WriterContext {
+ Writer getWriter();
+ }
+
+ public interface WriterCallback {
+ void preStripeWrite(WriterContext context) throws IOException;
+ void preFooterWrite(WriterContext context) throws IOException;
+ }
+
+ /**
+ * Options for creating ORC file writers.
+ */
+ public static class WriterOptions {
+ private final Configuration configuration;
+ private FileSystem fileSystemValue = null;
+ private TypeDescription schema = null;
+ private long stripeSizeValue;
+ private long blockSizeValue;
+ private int rowIndexStrideValue;
+ private int bufferSizeValue;
+ private boolean blockPaddingValue;
+ private CompressionKind compressValue;
+ private MemoryManager memoryManagerValue;
+ private Version versionValue;
+ private WriterCallback callback;
+ private EncodingStrategy encodingStrategy;
+ private CompressionStrategy compressionStrategy;
+ private double paddingTolerance;
+ private String bloomFilterColumns;
+ private double bloomFilterFpp;
+
+ protected WriterOptions(Properties tableProperties, Configuration conf) {
+ configuration = conf;
+ memoryManagerValue = getStaticMemoryManager(conf);
+ stripeSizeValue = OrcConf.STRIPE_SIZE.getLong(tableProperties, conf);
+ blockSizeValue = OrcConf.BLOCK_SIZE.getLong(tableProperties, conf);
+ rowIndexStrideValue =
+ (int) OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf);
+ bufferSizeValue = (int) OrcConf.BUFFER_SIZE.getLong(tableProperties,
+ conf);
+ blockPaddingValue =
+ OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf);
+ compressValue =
+ CompressionKind.valueOf(OrcConf.COMPRESS.getString(tableProperties,
+ conf));
+ String versionName = OrcConf.WRITE_FORMAT.getString(tableProperties,
+ conf);
+ versionValue = Version.byName(versionName);
+ String enString = OrcConf.ENCODING_STRATEGY.getString(tableProperties,
+ conf);
+ encodingStrategy = EncodingStrategy.valueOf(enString);
+
+ String compString =
+ OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf);
+ compressionStrategy = CompressionStrategy.valueOf(compString);
+
+ paddingTolerance =
+ OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf);
+
+ bloomFilterColumns = OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties,
+ conf);
+ bloomFilterFpp = OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties,
+ conf);
+ }
+
+ /**
+ * Provide the filesystem for the path, if the client has it available.
+ * If it is not provided, it will be found from the path.
+ */
+ public WriterOptions fileSystem(FileSystem value) {
+ fileSystemValue = value;
+ return this;
+ }
+
+ /**
+ * Set the stripe size for the file. The writer stores the contents of the
+ * stripe in memory until this memory limit is reached and the stripe
+ * is flushed to the HDFS file and the next stripe started.
+ */
+ public WriterOptions stripeSize(long value) {
+ stripeSizeValue = value;
+ return this;
+ }
+
+ /**
+ * Set the file system block size for the file. For optimal performance,
+ * set the block size to be multiple factors of stripe size.
+ */
+ public WriterOptions blockSize(long value) {
+ blockSizeValue = value;
+ return this;
+ }
+
+ /**
+ * Set the distance between entries in the row index. The minimum value is
+ * 1000 to prevent the index from overwhelming the data. If the stride is
+ * set to 0, no indexes will be included in the file.
+ */
+ public WriterOptions rowIndexStride(int value) {
+ rowIndexStrideValue = value;
+ return this;
+ }
+
+ /**
+ * The size of the memory buffers used for compressing and storing the
+ * stripe in memory.
+ */
+ public WriterOptions bufferSize(int value) {
+ bufferSizeValue = value;
+ return this;
+ }
+
+ /**
+ * Sets whether the HDFS blocks are padded to prevent stripes from
+ * straddling blocks. Padding improves locality and thus the speed of
+ * reading, but costs space.
+ */
+ public WriterOptions blockPadding(boolean value) {
+ blockPaddingValue = value;
+ return this;
+ }
+
+ /**
+ * Sets the encoding strategy that is used to encode the data.
+ */
+ public WriterOptions encodingStrategy(EncodingStrategy strategy) {
+ encodingStrategy = strategy;
+ return this;
+ }
+
+ /**
+ * Sets the tolerance for block padding as a percentage of stripe size.
+ */
+ public WriterOptions paddingTolerance(double value) {
+ paddingTolerance = value;
+ return this;
+ }
+
+ /**
+ * Comma separated values of column names for which bloom filter is to be created.
+ */
+ public WriterOptions bloomFilterColumns(String columns) {
+ bloomFilterColumns = columns;
+ return this;
+ }
+
+ /**
+ * Specify the false positive probability for bloom filter.
+ * @param fpp - false positive probability
+ * @return this
+ */
+ public WriterOptions bloomFilterFpp(double fpp) {
+ bloomFilterFpp = fpp;
+ return this;
+ }
+
+ /**
+ * Sets the generic compression that is used to compress the data.
+ */
+ public WriterOptions compress(CompressionKind value) {
+ compressValue = value;
+ return this;
+ }
+
+ /**
+ * Set the schema for the file. This is a required parameter.
+ * @param schema the schema for the file.
+ * @return this
+ */
+ public WriterOptions setSchema(TypeDescription schema) {
+ this.schema = schema;
+ return this;
+ }
+
+ /**
+ * Sets the version of the file that will be written.
+ */
+ public WriterOptions version(Version value) {
+ versionValue = value;
+ return this;
+ }
+
+ /**
+ * Add a listener for when the stripe and file are about to be closed.
+ * @param callback the object to be called when the stripe is closed
+ * @return this
+ */
+ public WriterOptions callback(WriterCallback callback) {
+ this.callback = callback;
+ return this;
+ }
+
+ /**
+ * A package local option to set the memory manager.
+ */
+ protected WriterOptions memory(MemoryManager value) {
+ memoryManagerValue = value;
+ return this;
+ }
+
+ public boolean getBlockPadding() {
+ return blockPaddingValue;
+ }
+
+ public long getBlockSize() {
+ return blockSizeValue;
+ }
+
+ public String getBloomFilterColumns() {
+ return bloomFilterColumns;
+ }
+
+ public FileSystem getFileSystem() {
+ return fileSystemValue;
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public TypeDescription getSchema() {
+ return schema;
+ }
+
+ public long getStripeSize() {
+ return stripeSizeValue;
+ }
+
+ public CompressionKind getCompress() {
+ return compressValue;
+ }
+
+ public WriterCallback getCallback() {
+ return callback;
+ }
+
+ public Version getVersion() {
+ return versionValue;
+ }
+
+ public MemoryManager getMemoryManager() {
+ return memoryManagerValue;
+ }
+
+ public int getBufferSize() {
+ return bufferSizeValue;
+ }
+
+ public int getRowIndexStride() {
+ return rowIndexStrideValue;
+ }
+
+ public CompressionStrategy getCompressionStrategy() {
+ return compressionStrategy;
+ }
+
+ public EncodingStrategy getEncodingStrategy() {
+ return encodingStrategy;
+ }
+
+ public double getPaddingTolerance() {
+ return paddingTolerance;
+ }
+
+ public double getBloomFilterFpp() {
+ return bloomFilterFpp;
+ }
+ }
+
+ /**
+ * Create a set of writer options based on a configuration.
+ * @param conf the configuration to use for values
+ * @return A WriterOptions object that can be modified
+ */
+ public static WriterOptions writerOptions(Configuration conf) {
+ return new WriterOptions(null, conf);
+ }
+
+ /**
+ * Create a set of write options based on a set of table properties and
+ * configuration.
+ * @param tableProperties the properties of the table
+ * @param conf the configuration of the query
+ * @return a WriterOptions object that can be modified
+ */
+ public static WriterOptions writerOptions(Properties tableProperties,
+ Configuration conf) {
+ return new WriterOptions(tableProperties, conf);
+ }
+
+ private static ThreadLocal<MemoryManager> memoryManager = null;
+
+ private static synchronized MemoryManager getStaticMemoryManager(
+ final Configuration conf) {
+ if (memoryManager == null) {
+ memoryManager = new ThreadLocal<MemoryManager>() {
+ @Override
+ protected MemoryManager initialValue() {
+ return new MemoryManager(conf);
+ }
+ };
+ }
+ return memoryManager.get();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/OrcUtils.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/OrcUtils.java b/orc/src/java/org/apache/orc/OrcUtils.java
new file mode 100644
index 0000000..2e93254
--- /dev/null
+++ b/orc/src/java/org/apache/orc/OrcUtils.java
@@ -0,0 +1,452 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+
+public class OrcUtils {
+
+ /**
+ * Returns selected columns as a boolean array with true value set for specified column names.
+ * The result will contain number of elements equal to flattened number of columns.
+ * For example:
+ * selectedColumns - a,b,c
+ * allColumns - a,b,c,d
+ * If column c is a complex type, say list<string> and other types are primitives then result will
+ * be [false, true, true, true, true, true, false]
+ * Index 0 is the root element of the struct which is set to false by default, index 1,2
+ * corresponds to columns a and b. Index 3,4 correspond to column c which is list<string> and
+ * index 5 correspond to column d. After flattening list<string> gets 2 columns.
+ *
+ * @param selectedColumns - comma separated list of selected column names
+ * @param schema - object schema
+ * @return - boolean array with true value set for the specified column names
+ */
+ public static boolean[] includeColumns(String selectedColumns,
+ TypeDescription schema) {
+ int numFlattenedCols = schema.getMaximumId();
+ boolean[] results = new boolean[numFlattenedCols + 1];
+ if ("*".equals(selectedColumns)) {
+ Arrays.fill(results, true);
+ return results;
+ }
+ if (selectedColumns != null &&
+ schema.getCategory() == TypeDescription.Category.STRUCT) {
+ List<String> fieldNames = schema.getFieldNames();
+ List<TypeDescription> fields = schema.getChildren();
+ for (String column: selectedColumns.split((","))) {
+ TypeDescription col = findColumn(column, fieldNames, fields);
+ if (col != null) {
+ for(int i=col.getId(); i <= col.getMaximumId(); ++i) {
+ results[i] = true;
+ }
+ }
+ }
+ }
+ return results;
+ }
+
+ private static TypeDescription findColumn(String columnName,
+ List<String> fieldNames,
+ List<TypeDescription> fields) {
+ int i = 0;
+ for(String fieldName: fieldNames) {
+ if (fieldName.equalsIgnoreCase(columnName)) {
+ return fields.get(i);
+ } else {
+ i += 1;
+ }
+ }
+ return null;
+ }
+
+ public static List<OrcProto.Type> getOrcTypes(TypeDescription typeDescr) {
+ List<OrcProto.Type> result = Lists.newArrayList();
+ appendOrcTypes(result, typeDescr);
+ return result;
+ }
+
+ private static void appendOrcTypes(List<OrcProto.Type> result, TypeDescription typeDescr) {
+ OrcProto.Type.Builder type = OrcProto.Type.newBuilder();
+ List<TypeDescription> children = typeDescr.getChildren();
+ switch (typeDescr.getCategory()) {
+ case BOOLEAN:
+ type.setKind(OrcProto.Type.Kind.BOOLEAN);
+ break;
+ case BYTE:
+ type.setKind(OrcProto.Type.Kind.BYTE);
+ break;
+ case SHORT:
+ type.setKind(OrcProto.Type.Kind.SHORT);
+ break;
+ case INT:
+ type.setKind(OrcProto.Type.Kind.INT);
+ break;
+ case LONG:
+ type.setKind(OrcProto.Type.Kind.LONG);
+ break;
+ case FLOAT:
+ type.setKind(OrcProto.Type.Kind.FLOAT);
+ break;
+ case DOUBLE:
+ type.setKind(OrcProto.Type.Kind.DOUBLE);
+ break;
+ case STRING:
+ type.setKind(OrcProto.Type.Kind.STRING);
+ break;
+ case CHAR:
+ type.setKind(OrcProto.Type.Kind.CHAR);
+ type.setMaximumLength(typeDescr.getMaxLength());
+ break;
+ case VARCHAR:
+ type.setKind(OrcProto.Type.Kind.VARCHAR);
+ type.setMaximumLength(typeDescr.getMaxLength());
+ break;
+ case BINARY:
+ type.setKind(OrcProto.Type.Kind.BINARY);
+ break;
+ case TIMESTAMP:
+ type.setKind(OrcProto.Type.Kind.TIMESTAMP);
+ break;
+ case DATE:
+ type.setKind(OrcProto.Type.Kind.DATE);
+ break;
+ case DECIMAL:
+ type.setKind(OrcProto.Type.Kind.DECIMAL);
+ type.setPrecision(typeDescr.getPrecision());
+ type.setScale(typeDescr.getScale());
+ break;
+ case LIST:
+ type.setKind(OrcProto.Type.Kind.LIST);
+ type.addSubtypes(children.get(0).getId());
+ break;
+ case MAP:
+ type.setKind(OrcProto.Type.Kind.MAP);
+ for(TypeDescription t: children) {
+ type.addSubtypes(t.getId());
+ }
+ break;
+ case STRUCT:
+ type.setKind(OrcProto.Type.Kind.STRUCT);
+ for(TypeDescription t: children) {
+ type.addSubtypes(t.getId());
+ }
+ for(String field: typeDescr.getFieldNames()) {
+ type.addFieldNames(field);
+ }
+ break;
+ case UNION:
+ type.setKind(OrcProto.Type.Kind.UNION);
+ for(TypeDescription t: children) {
+ type.addSubtypes(t.getId());
+ }
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown category: " +
+ typeDescr.getCategory());
+ }
+ result.add(type.build());
+ if (children != null) {
+ for(TypeDescription child: children) {
+ appendOrcTypes(result, child);
+ }
+ }
+ }
+
+ /**
+ * NOTE: This method ignores the subtype numbers in the TypeDescription rebuilds the subtype
+ * numbers based on the length of the result list being appended.
+ *
+ * @param result
+ * @param typeDescr
+ */
+ public static void appendOrcTypesRebuildSubtypes(List<OrcProto.Type> result,
+ TypeDescription typeDescr) {
+
+ int subtype = result.size();
+ OrcProto.Type.Builder type = OrcProto.Type.newBuilder();
+ boolean needsAdd = true;
+ List<TypeDescription> children = typeDescr.getChildren();
+ switch (typeDescr.getCategory()) {
+ case BOOLEAN:
+ type.setKind(OrcProto.Type.Kind.BOOLEAN);
+ break;
+ case BYTE:
+ type.setKind(OrcProto.Type.Kind.BYTE);
+ break;
+ case SHORT:
+ type.setKind(OrcProto.Type.Kind.SHORT);
+ break;
+ case INT:
+ type.setKind(OrcProto.Type.Kind.INT);
+ break;
+ case LONG:
+ type.setKind(OrcProto.Type.Kind.LONG);
+ break;
+ case FLOAT:
+ type.setKind(OrcProto.Type.Kind.FLOAT);
+ break;
+ case DOUBLE:
+ type.setKind(OrcProto.Type.Kind.DOUBLE);
+ break;
+ case STRING:
+ type.setKind(OrcProto.Type.Kind.STRING);
+ break;
+ case CHAR:
+ type.setKind(OrcProto.Type.Kind.CHAR);
+ type.setMaximumLength(typeDescr.getMaxLength());
+ break;
+ case VARCHAR:
+ type.setKind(OrcProto.Type.Kind.VARCHAR);
+ type.setMaximumLength(typeDescr.getMaxLength());
+ break;
+ case BINARY:
+ type.setKind(OrcProto.Type.Kind.BINARY);
+ break;
+ case TIMESTAMP:
+ type.setKind(OrcProto.Type.Kind.TIMESTAMP);
+ break;
+ case DATE:
+ type.setKind(OrcProto.Type.Kind.DATE);
+ break;
+ case DECIMAL:
+ type.setKind(OrcProto.Type.Kind.DECIMAL);
+ type.setPrecision(typeDescr.getPrecision());
+ type.setScale(typeDescr.getScale());
+ break;
+ case LIST:
+ type.setKind(OrcProto.Type.Kind.LIST);
+ type.addSubtypes(++subtype);
+ result.add(type.build());
+ needsAdd = false;
+ appendOrcTypesRebuildSubtypes(result, children.get(0));
+ break;
+ case MAP:
+ {
+ // Make room for MAP type.
+ result.add(null);
+
+ // Add MAP type pair in order to determine their subtype values.
+ appendOrcTypesRebuildSubtypes(result, children.get(0));
+ int subtype2 = result.size();
+ appendOrcTypesRebuildSubtypes(result, children.get(1));
+ type.setKind(OrcProto.Type.Kind.MAP);
+ type.addSubtypes(subtype + 1);
+ type.addSubtypes(subtype2);
+ result.set(subtype, type.build());
+ needsAdd = false;
+ }
+ break;
+ case STRUCT:
+ {
+ List<String> fieldNames = typeDescr.getFieldNames();
+
+ // Make room for STRUCT type.
+ result.add(null);
+
+ List<Integer> fieldSubtypes = new ArrayList<Integer>(fieldNames.size());
+ for(TypeDescription child: children) {
+ int fieldSubtype = result.size();
+ fieldSubtypes.add(fieldSubtype);
+ appendOrcTypesRebuildSubtypes(result, child);
+ }
+
+ type.setKind(OrcProto.Type.Kind.STRUCT);
+
+ for (int i = 0 ; i < fieldNames.size(); i++) {
+ type.addSubtypes(fieldSubtypes.get(i));
+ type.addFieldNames(fieldNames.get(i));
+ }
+ result.set(subtype, type.build());
+ needsAdd = false;
+ }
+ break;
+ case UNION:
+ {
+ // Make room for UNION type.
+ result.add(null);
+
+ List<Integer> unionSubtypes = new ArrayList<Integer>(children.size());
+ for(TypeDescription child: children) {
+ int unionSubtype = result.size();
+ unionSubtypes.add(unionSubtype);
+ appendOrcTypesRebuildSubtypes(result, child);
+ }
+
+ type.setKind(OrcProto.Type.Kind.UNION);
+ for (int i = 0 ; i < children.size(); i++) {
+ type.addSubtypes(unionSubtypes.get(i));
+ }
+ result.set(subtype, type.build());
+ needsAdd = false;
+ }
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown category: " + typeDescr.getCategory());
+ }
+ if (needsAdd) {
+ result.add(type.build());
+ }
+ }
+
+ /**
+ * NOTE: This method ignores the subtype numbers in the OrcProto.Type rebuilds the subtype
+ * numbers based on the length of the result list being appended.
+ *
+ * @param result
+ * @param types
+ * @param columnId
+ */
+ public static int appendOrcTypesRebuildSubtypes(List<OrcProto.Type> result,
+ List<OrcProto.Type> types, int columnId) {
+
+ OrcProto.Type oldType = types.get(columnId++);
+
+ int subtype = result.size();
+ OrcProto.Type.Builder builder = OrcProto.Type.newBuilder();
+ boolean needsAdd = true;
+ switch (oldType.getKind()) {
+ case BOOLEAN:
+ builder.setKind(OrcProto.Type.Kind.BOOLEAN);
+ break;
+ case BYTE:
+ builder.setKind(OrcProto.Type.Kind.BYTE);
+ break;
+ case SHORT:
+ builder.setKind(OrcProto.Type.Kind.SHORT);
+ break;
+ case INT:
+ builder.setKind(OrcProto.Type.Kind.INT);
+ break;
+ case LONG:
+ builder.setKind(OrcProto.Type.Kind.LONG);
+ break;
+ case FLOAT:
+ builder.setKind(OrcProto.Type.Kind.FLOAT);
+ break;
+ case DOUBLE:
+ builder.setKind(OrcProto.Type.Kind.DOUBLE);
+ break;
+ case STRING:
+ builder.setKind(OrcProto.Type.Kind.STRING);
+ break;
+ case CHAR:
+ builder.setKind(OrcProto.Type.Kind.CHAR);
+ builder.setMaximumLength(oldType.getMaximumLength());
+ break;
+ case VARCHAR:
+ builder.setKind(OrcProto.Type.Kind.VARCHAR);
+ builder.setMaximumLength(oldType.getMaximumLength());
+ break;
+ case BINARY:
+ builder.setKind(OrcProto.Type.Kind.BINARY);
+ break;
+ case TIMESTAMP:
+ builder.setKind(OrcProto.Type.Kind.TIMESTAMP);
+ break;
+ case DATE:
+ builder.setKind(OrcProto.Type.Kind.DATE);
+ break;
+ case DECIMAL:
+ builder.setKind(OrcProto.Type.Kind.DECIMAL);
+ builder.setPrecision(oldType.getPrecision());
+ builder.setScale(oldType.getScale());
+ break;
+ case LIST:
+ builder.setKind(OrcProto.Type.Kind.LIST);
+ builder.addSubtypes(++subtype);
+ result.add(builder.build());
+ needsAdd = false;
+ columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
+ break;
+ case MAP:
+ {
+ // Make room for MAP type.
+ result.add(null);
+
+ // Add MAP type pair in order to determine their subtype values.
+ columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
+ int subtype2 = result.size();
+ columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
+ builder.setKind(OrcProto.Type.Kind.MAP);
+ builder.addSubtypes(subtype + 1);
+ builder.addSubtypes(subtype2);
+ result.set(subtype, builder.build());
+ needsAdd = false;
+ }
+ break;
+ case STRUCT:
+ {
+ List<String> fieldNames = oldType.getFieldNamesList();
+
+ // Make room for STRUCT type.
+ result.add(null);
+
+ List<Integer> fieldSubtypes = new ArrayList<Integer>(fieldNames.size());
+ for(int i = 0 ; i < fieldNames.size(); i++) {
+ int fieldSubtype = result.size();
+ fieldSubtypes.add(fieldSubtype);
+ columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
+ }
+
+ builder.setKind(OrcProto.Type.Kind.STRUCT);
+
+ for (int i = 0 ; i < fieldNames.size(); i++) {
+ builder.addSubtypes(fieldSubtypes.get(i));
+ builder.addFieldNames(fieldNames.get(i));
+ }
+ result.set(subtype, builder.build());
+ needsAdd = false;
+ }
+ break;
+ case UNION:
+ {
+ int subtypeCount = oldType.getSubtypesCount();
+
+ // Make room for UNION type.
+ result.add(null);
+
+ List<Integer> unionSubtypes = new ArrayList<Integer>(subtypeCount);
+ for(int i = 0 ; i < subtypeCount; i++) {
+ int unionSubtype = result.size();
+ unionSubtypes.add(unionSubtype);
+ columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
+ }
+
+ builder.setKind(OrcProto.Type.Kind.UNION);
+ for (int i = 0 ; i < subtypeCount; i++) {
+ builder.addSubtypes(unionSubtypes.get(i));
+ }
+ result.set(subtype, builder.build());
+ needsAdd = false;
+ }
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown category: " + oldType.getKind());
+ }
+ if (needsAdd) {
+ result.add(builder.build());
+ }
+ return columnId;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/Reader.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/Reader.java b/orc/src/java/org/apache/orc/Reader.java
new file mode 100644
index 0000000..be722b5
--- /dev/null
+++ b/orc/src/java/org/apache/orc/Reader.java
@@ -0,0 +1,363 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.orc.impl.MetadataReader;
+
+/**
+ * The interface for reading ORC files.
+ *
+ * One Reader can support multiple concurrent RecordReader.
+ */
+public interface Reader {
+
+ /**
+ * Get the number of rows in the file.
+ * @return the number of rows
+ */
+ long getNumberOfRows();
+
+ /**
+ * Get the deserialized data size of the file
+ * @return raw data size
+ */
+ long getRawDataSize();
+
+ /**
+ * Get the deserialized data size of the specified columns
+ * @param colNames
+ * @return raw data size of columns
+ */
+ long getRawDataSizeOfColumns(List<String> colNames);
+
+ /**
+ * Get the deserialized data size of the specified columns ids
+ * @param colIds - internal column id (check orcfiledump for column ids)
+ * @return raw data size of columns
+ */
+ long getRawDataSizeFromColIndices(List<Integer> colIds);
+
+ /**
+ * Get the user metadata keys.
+ * @return the set of metadata keys
+ */
+ List<String> getMetadataKeys();
+
+ /**
+ * Get a user metadata value.
+ * @param key a key given by the user
+ * @return the bytes associated with the given key
+ */
+ ByteBuffer getMetadataValue(String key);
+
+ /**
+ * Did the user set the given metadata value.
+ * @param key the key to check
+ * @return true if the metadata value was set
+ */
+ boolean hasMetadataValue(String key);
+
+ /**
+ * Get the compression kind.
+ * @return the kind of compression in the file
+ */
+ CompressionKind getCompressionKind();
+
+ /**
+ * Get the buffer size for the compression.
+ * @return number of bytes to buffer for the compression codec.
+ */
+ int getCompressionSize();
+
+ /**
+ * Get the number of rows per a entry in the row index.
+ * @return the number of rows per an entry in the row index or 0 if there
+ * is no row index.
+ */
+ int getRowIndexStride();
+
+ /**
+ * Get the list of stripes.
+ * @return the information about the stripes in order
+ */
+ List<StripeInformation> getStripes();
+
+ /**
+ * Get the length of the file.
+ * @return the number of bytes in the file
+ */
+ long getContentLength();
+
+ /**
+ * Get the statistics about the columns in the file.
+ * @return the information about the column
+ */
+ ColumnStatistics[] getStatistics();
+
+ /**
+ * Get the list of types contained in the file. The root type is the first
+ * type in the list.
+ * @return the list of flattened types
+ */
+ List<OrcProto.Type> getTypes();
+
+ /**
+ * Get the file format version.
+ */
+ OrcFile.Version getFileVersion();
+
+ /**
+ * Get the version of the writer of this file.
+ */
+ OrcFile.WriterVersion getWriterVersion();
+
+ /**
+ * Options for creating a RecordReader.
+ */
+ public static class Options {
+ private boolean[] include;
+ private long offset = 0;
+ private long length = Long.MAX_VALUE;
+ private SearchArgument sarg = null;
+ private String[] columnNames = null;
+ private Boolean useZeroCopy = null;
+ private Boolean skipCorruptRecords = null;
+ private TypeDescription schema = null;
+
+ /**
+ * Set the list of columns to read.
+ * @param include a list of columns to read
+ * @return this
+ */
+ public Options include(boolean[] include) {
+ this.include = include;
+ return this;
+ }
+
+ /**
+ * Set the range of bytes to read
+ * @param offset the starting byte offset
+ * @param length the number of bytes to read
+ * @return this
+ */
+ public Options range(long offset, long length) {
+ this.offset = offset;
+ this.length = length;
+ return this;
+ }
+
+ /**
+ * Set the schema on read type description.
+ */
+ public Options schema(TypeDescription schema) {
+ this.schema = schema;
+ return this;
+ }
+
+ /**
+ * Set search argument for predicate push down.
+ * @param sarg the search argument
+ * @param columnNames the column names for
+ * @return this
+ */
+ public Options searchArgument(SearchArgument sarg, String[] columnNames) {
+ this.sarg = sarg;
+ this.columnNames = columnNames;
+ return this;
+ }
+
+ /**
+ * Set whether to use zero copy from HDFS.
+ * @param value the new zero copy flag
+ * @return this
+ */
+ public Options useZeroCopy(boolean value) {
+ this.useZeroCopy = value;
+ return this;
+ }
+
+ /**
+ * Set whether to skip corrupt records.
+ * @param value the new skip corrupt records flag
+ * @return this
+ */
+ public Options skipCorruptRecords(boolean value) {
+ this.skipCorruptRecords = value;
+ return this;
+ }
+
+ public boolean[] getInclude() {
+ return include;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public long getLength() {
+ return length;
+ }
+
+ public TypeDescription getSchema() {
+ return schema;
+ }
+
+ public SearchArgument getSearchArgument() {
+ return sarg;
+ }
+
+ public String[] getColumnNames() {
+ return columnNames;
+ }
+
+ public long getMaxOffset() {
+ long result = offset + length;
+ if (result < 0) {
+ result = Long.MAX_VALUE;
+ }
+ return result;
+ }
+
+ public Boolean getUseZeroCopy() {
+ return useZeroCopy;
+ }
+
+ public Boolean getSkipCorruptRecords() {
+ return skipCorruptRecords;
+ }
+
+ public Options clone() {
+ Options result = new Options();
+ result.include = include;
+ result.offset = offset;
+ result.length = length;
+ result.sarg = sarg;
+ result.schema = schema;
+ result.columnNames = columnNames;
+ result.useZeroCopy = useZeroCopy;
+ result.skipCorruptRecords = skipCorruptRecords;
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("{include: ");
+ if (include == null) {
+ buffer.append("null");
+ } else {
+ buffer.append("[");
+ for(int i=0; i < include.length; ++i) {
+ if (i != 0) {
+ buffer.append(", ");
+ }
+ buffer.append(include[i]);
+ }
+ buffer.append("]");
+ }
+ buffer.append(", offset: ");
+ buffer.append(offset);
+ buffer.append(", length: ");
+ buffer.append(length);
+ if (sarg != null) {
+ buffer.append(", sarg: ");
+ buffer.append(sarg.toString());
+ buffer.append(", columns: [");
+ for(int i=0; i < columnNames.length; ++i) {
+ if (i != 0) {
+ buffer.append(", ");
+ }
+ buffer.append("'");
+ buffer.append(columnNames[i]);
+ buffer.append("'");
+ }
+ buffer.append("]");
+ }
+ if (schema != null) {
+ buffer.append(", schema: ");
+ schema.printToBuffer(buffer);
+ }
+ buffer.append("}");
+ return buffer.toString();
+ }
+ }
+
+ /**
+ * Create a RecordReader that reads everything with the default options.
+ * @return a new RecordReader
+ * @throws IOException
+ */
+ RecordReader rows() throws IOException;
+
+ /**
+ * Create a RecordReader that uses the options given.
+ * This method can't be named rows, because many callers used rows(null)
+ * before the rows() method was introduced.
+ * @param options the options to read with
+ * @return a new RecordReader
+ * @throws IOException
+ */
+ RecordReader rowsOptions(Options options) throws IOException;
+
+ /**
+ * @return Metadata reader used to read file metadata.
+ */
+ MetadataReader metadata() throws IOException;
+
+ /**
+ * @return List of integers representing version of the file, in order from major to minor.
+ */
+ List<Integer> getVersionList();
+
+ /**
+ * @return Gets the size of metadata, in bytes.
+ */
+ int getMetadataSize();
+
+ /**
+ * @return Stripe statistics, in original protobuf form.
+ */
+ List<OrcProto.StripeStatistics> getOrcProtoStripeStatistics();
+
+ /**
+ * @return Stripe statistics.
+ */
+ List<StripeStatistics> getStripeStatistics();
+
+ /**
+ * @return File statistics, in original protobuf form.
+ */
+ List<OrcProto.ColumnStatistics> getOrcProtoFileStatistics();
+
+ /**
+ * @param useZeroCopy Whether zero-copy read should be used.
+ * @return The default data reader that ORC is using to read bytes from disk.
+ */
+ DataReader createDefaultDataReader(boolean useZeroCopy);
+
+ /**
+ * @return Serialized file metadata read from disk for the purposes of caching, etc.
+ */
+ ByteBuffer getSerializedFileFooter();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/RecordReader.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/RecordReader.java b/orc/src/java/org/apache/orc/RecordReader.java
new file mode 100644
index 0000000..7229dda
--- /dev/null
+++ b/orc/src/java/org/apache/orc/RecordReader.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+/**
+ * A row-by-row iterator for ORC files.
+ */
+public interface RecordReader {
+ /**
+ * Read the next row batch. The size of the batch to read cannot be
+ * controlled by the callers. Caller need to look at
+ * VectorizedRowBatch.size of the retunred object to know the batch
+ * size read.
+ * @param previousBatch a row batch object that can be reused by the reader
+ * @return the row batch that was read. The batch will have a non-zero row
+ * count if the pointer isn't at the end of the file
+ * @throws java.io.IOException
+ */
+ VectorizedRowBatch nextBatch(VectorizedRowBatch previousBatch
+ ) throws IOException;
+
+ /**
+ * Get the row number of the row that will be returned by the following
+ * call to next().
+ * @return the row number from 0 to the number of rows in the file
+ * @throws java.io.IOException
+ */
+ long getRowNumber() throws IOException;
+
+ /**
+ * Get the progress of the reader through the rows.
+ * @return a fraction between 0.0 and 1.0 of rows read
+ * @throws java.io.IOException
+ */
+ float getProgress() throws IOException;
+
+ /**
+ * Release the resources associated with the given reader.
+ * @throws java.io.IOException
+ */
+ void close() throws IOException;
+
+ /**
+ * Seek to a particular row number.
+ */
+ void seekToRow(long rowCount) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/StringColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/StringColumnStatistics.java b/orc/src/java/org/apache/orc/StringColumnStatistics.java
new file mode 100644
index 0000000..5a868d0
--- /dev/null
+++ b/orc/src/java/org/apache/orc/StringColumnStatistics.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import org.apache.orc.ColumnStatistics;
+
+/**
+ * Statistics for string columns.
+ */
+public interface StringColumnStatistics extends ColumnStatistics {
+ /**
+ * Get the minimum string.
+ * @return the minimum
+ */
+ String getMinimum();
+
+ /**
+ * Get the maximum string.
+ * @return the maximum
+ */
+ String getMaximum();
+
+ /**
+ * Get the total length of all strings
+ * @return the sum (total length)
+ */
+ long getSum();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/StripeInformation.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/StripeInformation.java b/orc/src/java/org/apache/orc/StripeInformation.java
new file mode 100644
index 0000000..38f7eba
--- /dev/null
+++ b/orc/src/java/org/apache/orc/StripeInformation.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+/**
+ * Information about the stripes in an ORC file that is provided by the Reader.
+ */
+public interface StripeInformation {
+ /**
+ * Get the byte offset of the start of the stripe.
+ * @return the bytes from the start of the file
+ */
+ long getOffset();
+
+ /**
+ * Get the total length of the stripe in bytes.
+ * @return the number of bytes in the stripe
+ */
+ long getLength();
+
+ /**
+ * Get the length of the stripe's indexes.
+ * @return the number of bytes in the index
+ */
+ long getIndexLength();
+
+ /**
+ * Get the length of the stripe's data.
+ * @return the number of bytes in the stripe
+ */
+ long getDataLength();
+
+ /**
+ * Get the length of the stripe's tail section, which contains its index.
+ * @return the number of bytes in the tail
+ */
+ long getFooterLength();
+
+ /**
+ * Get the number of rows in the stripe.
+ * @return a count of the number of rows
+ */
+ long getNumberOfRows();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/StripeStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/StripeStatistics.java b/orc/src/java/org/apache/orc/StripeStatistics.java
new file mode 100644
index 0000000..8fc91cb
--- /dev/null
+++ b/orc/src/java/org/apache/orc/StripeStatistics.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import org.apache.orc.impl.ColumnStatisticsImpl;
+
+import java.util.List;
+
+public class StripeStatistics {
+ private final List<OrcProto.ColumnStatistics> cs;
+
+ public StripeStatistics(List<OrcProto.ColumnStatistics> list) {
+ this.cs = list;
+ }
+
+ /**
+ * Return list of column statistics
+ *
+ * @return column stats
+ */
+ public ColumnStatistics[] getColumnStatistics() {
+ ColumnStatistics[] result = new ColumnStatistics[cs.size()];
+ for (int i = 0; i < result.length; ++i) {
+ result[i] = ColumnStatisticsImpl.deserialize(cs.get(i));
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/src/java/org/apache/orc/TimestampColumnStatistics.java
----------------------------------------------------------------------
diff --git a/orc/src/java/org/apache/orc/TimestampColumnStatistics.java b/orc/src/java/org/apache/orc/TimestampColumnStatistics.java
new file mode 100644
index 0000000..27dc49f
--- /dev/null
+++ b/orc/src/java/org/apache/orc/TimestampColumnStatistics.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import java.sql.Timestamp;
+
+/**
+ * Statistics for Timestamp columns.
+ */
+public interface TimestampColumnStatistics extends ColumnStatistics {
+ /**
+ * Get the minimum value for the column.
+ * @return minimum value
+ */
+ Timestamp getMinimum();
+
+ /**
+ * Get the maximum value for the column.
+ * @return maximum value
+ */
+ Timestamp getMaximum();
+}
[16/16] hive git commit: HIVE-11890. Create ORC submodue. (omalley
reviewed by prasanthj)
Posted by om...@apache.org.
HIVE-11890. Create ORC submodue. (omalley reviewed by prasanthj)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9c7a78ee
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9c7a78ee
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9c7a78ee
Branch: refs/heads/master
Commit: 9c7a78ee36d43637237bc9af2df6dbd108f5c43e
Parents: 3e3d966
Author: Owen O'Malley <om...@apache.org>
Authored: Mon Nov 2 09:00:33 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Dec 11 15:27:05 2015 -0800
----------------------------------------------------------------------
common/pom.xml | 5 +
.../hadoop/hive/common/DiskRangeInfo.java | 59 -
.../hive/hcatalog/mapreduce/SpecialCases.java | 2 +-
.../llap/io/decode/OrcEncodedDataConsumer.java | 4 +-
.../llap/io/encoded/OrcEncodedDataReader.java | 12 +-
.../hive/llap/io/metadata/OrcFileMetadata.java | 41 +-
.../llap/io/metadata/OrcStripeMetadata.java | 46 +-
.../TestIncrementalObjectSizeEstimator.java | 91 +-
orc/pom.xml | 136 +
.../protobuf-java/org/apache/orc/OrcProto.java | 19279 ++++++++++++++++
.../org/apache/orc/BinaryColumnStatistics.java | 27 +
.../org/apache/orc/BooleanColumnStatistics.java | 29 +
.../java/org/apache/orc/ColumnStatistics.java | 36 +
.../java/org/apache/orc/CompressionCodec.java | 69 +
.../java/org/apache/orc/CompressionKind.java | 27 +
orc/src/java/org/apache/orc/DataReader.java | 58 +
.../org/apache/orc/DateColumnStatistics.java | 39 +
.../org/apache/orc/DecimalColumnStatistics.java | 46 +
.../org/apache/orc/DoubleColumnStatistics.java | 46 +
orc/src/java/org/apache/orc/FileMetaInfo.java | 64 +
orc/src/java/org/apache/orc/FileMetadata.java | 64 +
.../org/apache/orc/IntegerColumnStatistics.java | 52 +
orc/src/java/org/apache/orc/OrcConf.java | 191 +
orc/src/java/org/apache/orc/OrcFile.java | 514 +
orc/src/java/org/apache/orc/OrcUtils.java | 452 +
orc/src/java/org/apache/orc/Reader.java | 363 +
orc/src/java/org/apache/orc/RecordReader.java | 66 +
.../org/apache/orc/StringColumnStatistics.java | 43 +
.../java/org/apache/orc/StripeInformation.java | 59 +
.../java/org/apache/orc/StripeStatistics.java | 44 +
.../apache/orc/TimestampColumnStatistics.java | 38 +
.../java/org/apache/orc/TypeDescription.java | 540 +
orc/src/java/org/apache/orc/Writer.java | 114 +
.../org/apache/orc/impl/BitFieldReader.java | 216 +
.../org/apache/orc/impl/BitFieldWriter.java | 73 +
.../java/org/apache/orc/impl/BufferChunk.java | 85 +
.../apache/orc/impl/ColumnStatisticsImpl.java | 1097 +
.../orc/impl/DirectDecompressionCodec.java | 28 +
.../org/apache/orc/impl/DynamicByteArray.java | 303 +
.../org/apache/orc/impl/DynamicIntArray.java | 142 +
.../java/org/apache/orc/impl/HadoopShims.java | 64 +
.../org/apache/orc/impl/HadoopShimsCurrent.java | 62 +
.../org/apache/orc/impl/HadoopShims_2_2.java | 36 +
orc/src/java/org/apache/orc/impl/InStream.java | 496 +
.../java/org/apache/orc/impl/IntegerReader.java | 67 +
.../java/org/apache/orc/impl/IntegerWriter.java | 47 +
.../java/org/apache/orc/impl/MemoryManager.java | 214 +
.../org/apache/orc/impl/MetadataReader.java | 34 +
.../org/apache/orc/impl/MetadataReaderImpl.java | 125 +
orc/src/java/org/apache/orc/impl/OrcIndex.java | 43 +
orc/src/java/org/apache/orc/impl/OutStream.java | 289 +
.../org/apache/orc/impl/PositionProvider.java | 26 +
.../org/apache/orc/impl/PositionRecorder.java | 25 +
.../apache/orc/impl/PositionedOutputStream.java | 39 +
.../java/org/apache/orc/impl/RedBlackTree.java | 311 +
.../apache/orc/impl/RunLengthByteReader.java | 150 +
.../apache/orc/impl/RunLengthByteWriter.java | 106 +
.../apache/orc/impl/RunLengthIntegerReader.java | 157 +
.../orc/impl/RunLengthIntegerReaderV2.java | 390 +
.../apache/orc/impl/RunLengthIntegerWriter.java | 143 +
.../orc/impl/RunLengthIntegerWriterV2.java | 831 +
.../org/apache/orc/impl/SerializationUtils.java | 1297 ++
.../orc/impl/SettableUncompressedStream.java | 44 +
.../java/org/apache/orc/impl/SnappyCodec.java | 108 +
.../java/org/apache/orc/impl/StreamName.java | 97 +
.../org/apache/orc/impl/StringRedBlackTree.java | 210 +
orc/src/java/org/apache/orc/impl/ZlibCodec.java | 169 +
orc/src/protobuf/orc_proto.proto | 220 +
.../org/apache/orc/impl/TestBitFieldReader.java | 145 +
.../test/org/apache/orc/impl/TestBitPack.java | 279 +
.../org/apache/orc/impl/TestDynamicArray.java | 90 +
.../test/org/apache/orc/impl/TestInStream.java | 314 +
.../orc/impl/TestIntegerCompressionReader.java | 130 +
.../org/apache/orc/impl/TestMemoryManager.java | 133 +
.../orc/impl/TestRunLengthByteReader.java | 143 +
.../orc/impl/TestRunLengthIntegerReader.java | 125 +
.../apache/orc/impl/TestSerializationUtils.java | 164 +
.../apache/orc/impl/TestStringRedBlackTree.java | 234 +
orc/src/test/org/apache/orc/impl/TestZlib.java | 56 +
pom.xml | 1 +
ql/pom.xml | 54 +-
.../apache/hadoop/hive/ql/io/orc/OrcProto.java | 19280 -----------------
.../hive/ql/exec/OrcFileMergeOperator.java | 4 +-
.../hive/ql/io/filters/BloomFilterIO.java | 2 +-
.../hive/ql/io/orc/BinaryColumnStatistics.java | 25 -
.../hadoop/hive/ql/io/orc/BitFieldReader.java | 212 -
.../hadoop/hive/ql/io/orc/BitFieldWriter.java | 69 -
.../hive/ql/io/orc/BooleanColumnStatistics.java | 27 -
.../hadoop/hive/ql/io/orc/ColumnStatistics.java | 36 -
.../hive/ql/io/orc/ColumnStatisticsImpl.java | 1082 -
.../hadoop/hive/ql/io/orc/CompressionCodec.java | 69 -
.../hadoop/hive/ql/io/orc/CompressionKind.java | 18 +-
.../hadoop/hive/ql/io/orc/DataReader.java | 58 -
.../hive/ql/io/orc/DateColumnStatistics.java | 37 -
.../hive/ql/io/orc/DecimalColumnStatistics.java | 45 -
.../ql/io/orc/DirectDecompressionCodec.java | 26 -
.../hive/ql/io/orc/DoubleColumnStatistics.java | 44 -
.../hadoop/hive/ql/io/orc/DynamicByteArray.java | 303 -
.../hadoop/hive/ql/io/orc/DynamicIntArray.java | 142 -
.../apache/hadoop/hive/ql/io/orc/FileDump.java | 20 +-
.../hadoop/hive/ql/io/orc/FileMetaInfo.java | 64 -
.../hadoop/hive/ql/io/orc/FileMetadata.java | 63 -
.../apache/hadoop/hive/ql/io/orc/InStream.java | 497 -
.../hive/ql/io/orc/IntegerColumnStatistics.java | 50 -
.../hadoop/hive/ql/io/orc/IntegerReader.java | 67 -
.../hadoop/hive/ql/io/orc/IntegerWriter.java | 47 -
.../hadoop/hive/ql/io/orc/JsonFileDump.java | 16 +-
.../hadoop/hive/ql/io/orc/MemoryManager.java | 213 -
.../hadoop/hive/ql/io/orc/MetadataReader.java | 34 -
.../hive/ql/io/orc/MetadataReaderImpl.java | 123 -
.../apache/hadoop/hive/ql/io/orc/OrcConf.java | 191 -
.../apache/hadoop/hive/ql/io/orc/OrcFile.java | 364 +-
.../hive/ql/io/orc/OrcFileKeyWrapper.java | 2 +
.../io/orc/OrcFileStripeMergeRecordReader.java | 4 +-
.../hive/ql/io/orc/OrcFileValueWrapper.java | 2 +
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 193 +-
.../hive/ql/io/orc/OrcNewInputFormat.java | 1 +
.../hadoop/hive/ql/io/orc/OrcNewSplit.java | 1 +
.../hadoop/hive/ql/io/orc/OrcOutputFormat.java | 5 +-
.../hive/ql/io/orc/OrcRawRecordMerger.java | 7 +-
.../apache/hadoop/hive/ql/io/orc/OrcSerde.java | 1 +
.../apache/hadoop/hive/ql/io/orc/OrcSplit.java | 2 +-
.../apache/hadoop/hive/ql/io/orc/OrcStruct.java | 1 +
.../apache/hadoop/hive/ql/io/orc/OrcUnion.java | 1 +
.../apache/hadoop/hive/ql/io/orc/OrcUtils.java | 629 -
.../apache/hadoop/hive/ql/io/orc/OutStream.java | 286 -
.../hadoop/hive/ql/io/orc/PositionProvider.java | 26 -
.../hadoop/hive/ql/io/orc/PositionRecorder.java | 25 -
.../hive/ql/io/orc/PositionedOutputStream.java | 38 -
.../apache/hadoop/hive/ql/io/orc/Reader.java | 319 +-
.../hadoop/hive/ql/io/orc/ReaderImpl.java | 50 +-
.../hadoop/hive/ql/io/orc/RecordReader.java | 38 +-
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 116 +-
.../hive/ql/io/orc/RecordReaderUtils.java | 12 +-
.../hadoop/hive/ql/io/orc/RedBlackTree.java | 309 -
.../hive/ql/io/orc/RunLengthByteReader.java | 150 -
.../hive/ql/io/orc/RunLengthByteWriter.java | 106 -
.../hive/ql/io/orc/RunLengthIntegerReader.java | 157 -
.../ql/io/orc/RunLengthIntegerReaderV2.java | 392 -
.../hive/ql/io/orc/RunLengthIntegerWriter.java | 143 -
.../ql/io/orc/RunLengthIntegerWriterV2.java | 831 -
.../hadoop/hive/ql/io/orc/SchemaEvolution.java | 12 +-
.../hive/ql/io/orc/SerializationUtils.java | 1291 --
.../ql/io/orc/SettableUncompressedStream.java | 43 -
.../hadoop/hive/ql/io/orc/SnappyCodec.java | 109 -
.../hadoop/hive/ql/io/orc/StreamName.java | 95 -
.../hive/ql/io/orc/StringColumnStatistics.java | 41 -
.../hive/ql/io/orc/StringRedBlackTree.java | 207 -
.../hive/ql/io/orc/StripeInformation.java | 59 -
.../hadoop/hive/ql/io/orc/StripeStatistics.java | 42 -
.../ql/io/orc/TimestampColumnStatistics.java | 38 -
.../hive/ql/io/orc/TreeReaderFactory.java | 14 +-
.../hadoop/hive/ql/io/orc/TypeDescription.java | 540 -
.../ql/io/orc/VectorizedOrcInputFormat.java | 4 +-
.../apache/hadoop/hive/ql/io/orc/Writer.java | 88 +-
.../hadoop/hive/ql/io/orc/WriterImpl.java | 57 +-
.../apache/hadoop/hive/ql/io/orc/ZlibCodec.java | 171 -
.../hive/ql/io/orc/encoded/EncodedReader.java | 9 +-
.../ql/io/orc/encoded/EncodedReaderImpl.java | 31 +-
.../orc/encoded/EncodedTreeReaderFactory.java | 99 +-
.../hadoop/hive/ql/io/orc/encoded/Reader.java | 6 +-
.../hive/ql/io/orc/encoded/ReaderImpl.java | 3 +-
.../hive/ql/io/orc/encoded/StreamUtils.java | 7 +-
.../ppr/PartitionExpressionForMetastore.java | 4 +-
.../hadoop/hive/ql/io/orc/orc_proto.proto | 220 -
.../ql/exec/vector/util/OrcFileGenerator.java | 2 +-
.../hive/ql/io/orc/TestBitFieldReader.java | 144 -
.../hadoop/hive/ql/io/orc/TestBitPack.java | 316 -
.../hive/ql/io/orc/TestColumnStatistics.java | 11 +-
.../hadoop/hive/ql/io/orc/TestDynamicArray.java | 87 -
.../hadoop/hive/ql/io/orc/TestInStream.java | 313 -
.../hive/ql/io/orc/TestInputOutputFormat.java | 2 +
.../ql/io/orc/TestIntegerCompressionReader.java | 129 -
.../hadoop/hive/ql/io/orc/TestJsonFileDump.java | 1 +
.../hive/ql/io/orc/TestMemoryManager.java | 132 -
.../hive/ql/io/orc/TestNewIntegerEncoding.java | 10 +-
.../hadoop/hive/ql/io/orc/TestOrcFile.java | 63 +-
.../hive/ql/io/orc/TestOrcNullOptimization.java | 7 +
.../hive/ql/io/orc/TestOrcRawRecordMerger.java | 9 +-
.../hive/ql/io/orc/TestOrcSerDeStats.java | 7 +
.../hive/ql/io/orc/TestRecordReaderImpl.java | 5 +-
.../hive/ql/io/orc/TestRunLengthByteReader.java | 142 -
.../ql/io/orc/TestRunLengthIntegerReader.java | 124 -
.../hive/ql/io/orc/TestSerializationUtils.java | 164 -
.../hadoop/hive/ql/io/orc/TestStreamName.java | 2 +
.../hive/ql/io/orc/TestStringDictionary.java | 7 +-
.../hive/ql/io/orc/TestStringRedBlackTree.java | 288 -
.../hive/ql/io/orc/TestTypeDescription.java | 1 +
.../hive/ql/io/orc/TestUnrolledBitPack.java | 1 +
.../hive/ql/io/orc/TestVectorOrcFile.java | 38 +-
.../hive/ql/io/orc/TestVectorizedORCReader.java | 1 -
.../apache/hadoop/hive/ql/io/orc/TestZlib.java | 55 -
.../hadoop/hive/serde2/io/DateWritable.java | 179 -
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 9 -
.../apache/hadoop/hive/shims/ZeroCopyShims.java | 42 -
.../apache/hadoop/hive/shims/HadoopShims.java | 13 -
.../hadoop/hive/common/DiskRangeInfo.java | 59 +
.../hadoop/hive/serde2/io/DateWritable.java | 179 +
198 files changed, 32974 insertions(+), 32101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index ee597cf..8141f75 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -44,6 +44,11 @@
<artifactId>hive-storage-api</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-orc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- inter-project -->
<dependency>
<groupId>commons-cli</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/common/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java b/common/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
deleted file mode 100644
index 86b838c..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.common.io.DiskRange;
-
-import com.google.common.collect.Lists;
-
-/**
- * Disk range information class containing disk ranges and total length.
- */
-public class DiskRangeInfo {
- List<DiskRange> diskRanges; // TODO: use DiskRangeList instead
- long totalLength;
-
- public DiskRangeInfo(int indexBaseOffset) {
- this.diskRanges = Lists.newArrayList();
- // Some data is missing from the stream for PPD uncompressed read (because index offset is
- // relative to the entire stream and we only read part of stream if RGs are filtered; unlike
- // with compressed data where PPD only filters CBs, so we always get full CB, and index offset
- // is relative to CB). To take care of the case when UncompressedStream goes seeking around by
- // its incorrect (relative to partial stream) index offset, we will increase the length by our
- // offset-relative-to-the-stream, and also account for it in buffers (see createDiskRangeInfo).
- // So, index offset now works; as long as noone seeks into this data before the RG (why would
- // they), everything works. This is hacky... Stream shouldn't depend on having all the data.
- this.totalLength = indexBaseOffset;
- }
-
- public void addDiskRange(DiskRange diskRange) {
- diskRanges.add(diskRange);
- totalLength += diskRange.getLength();
- }
-
- public List<DiskRange> getDiskRanges() {
- return diskRanges;
- }
-
- public long getTotalLength() {
- return totalLength;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java
index 756abf8..60af5c0 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
import org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat;
-import org.apache.hadoop.hive.ql.io.orc.OrcConf;
+import org.apache.orc.OrcConf;
import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
import org.apache.hadoop.hive.serde2.avro.AvroSerDe;
import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
index 5fd5226..2597848 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
@@ -28,16 +28,16 @@ import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.io.orc.CompressionCodec;
+import org.apache.orc.CompressionCodec;
import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer;
import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedTreeReaderFactory;
import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedTreeReaderFactory.SettableTreeReader;
import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
import org.apache.hadoop.hive.ql.io.orc.TreeReaderFactory;
import org.apache.hadoop.hive.ql.io.orc.WriterImpl;
+import org.apache.orc.OrcProto;
public class OrcEncodedDataConsumer
extends EncodedDataConsumer<OrcBatchKey, OrcEncodedColumnBatch> {
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index f9d06e9..729f1bd 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -55,14 +55,13 @@ import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
import org.apache.hadoop.hive.ql.exec.DDLTask;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.HdfsUtils;
-import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
-import org.apache.hadoop.hive.ql.io.orc.DataReader;
-import org.apache.hadoop.hive.ql.io.orc.MetadataReader;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.DataReader;
+import org.apache.orc.impl.MetadataReader;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.OrcFile.ReaderOptions;
-import org.apache.hadoop.hive.ql.io.orc.OrcConf;
+import org.apache.orc.OrcConf;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto;
import org.apache.hadoop.hive.ql.io.orc.OrcSplit;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
@@ -75,12 +74,13 @@ import org.apache.hadoop.hive.ql.io.orc.encoded.OrcCacheKey;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch;
import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.PoolFactory;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils;
-import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
+import org.apache.orc.StripeInformation;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.common.util.FixedSizedObjectPool;
+import org.apache.orc.OrcProto;
import org.apache.tez.common.CallableWithNdc;
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java
index 9ff3523..2e4e0c5 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.llap.io.metadata;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -27,19 +26,13 @@ import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator;
import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator;
import org.apache.hadoop.hive.llap.cache.EvictionDispatcher;
import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer;
-import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
-import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
-import org.apache.hadoop.hive.ql.io.orc.FileMetadata;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.FileMetadata;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.ql.io.orc.ReaderImpl.StripeInformationImpl;
-import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.OrcProto;
import com.google.common.annotations.VisibleForTesting;
@@ -50,9 +43,9 @@ import com.google.common.annotations.VisibleForTesting;
public final class OrcFileMetadata extends LlapCacheableBuffer implements FileMetadata {
private final List<StripeInformation> stripes;
private final List<Integer> versionList;
- private final List<StripeStatistics> stripeStats;
- private final List<Type> types;
- private final List<ColumnStatistics> fileStats;
+ private final List<OrcProto.StripeStatistics> stripeStats;
+ private final List<OrcProto.Type> types;
+ private final List<OrcProto.ColumnStatistics> fileStats;
private final long fileId;
private final CompressionKind compressionKind;
private final int rowIndexStride;
@@ -80,17 +73,17 @@ public final class OrcFileMetadata extends LlapCacheableBuffer implements FileMe
OrcFileMetadata ofm = new OrcFileMetadata(fileId);
ofm.stripes.add(new StripeInformationImpl(
OrcProto.StripeInformation.getDefaultInstance()));
- ofm.fileStats.add(ColumnStatistics.getDefaultInstance());
- ofm.stripeStats.add(StripeStatistics.newBuilder().addColStats(createStatsDummy()).build());
- ofm.types.add(Type.newBuilder().addFieldNames("a").addSubtypes(0).build());
+ ofm.fileStats.add(OrcProto.ColumnStatistics.getDefaultInstance());
+ ofm.stripeStats.add(OrcProto.StripeStatistics.newBuilder().addColStats(createStatsDummy()).build());
+ ofm.types.add(OrcProto.Type.newBuilder().addFieldNames("a").addSubtypes(0).build());
ofm.versionList.add(0);
return ofm;
}
- static ColumnStatistics.Builder createStatsDummy() {
- return ColumnStatistics.newBuilder().setBucketStatistics(
- BucketStatistics.newBuilder().addCount(0)).setStringStatistics(
- StringStatistics.newBuilder().setMaximum("zzz"));
+ static OrcProto.ColumnStatistics.Builder createStatsDummy() {
+ return OrcProto.ColumnStatistics.newBuilder().setBucketStatistics(
+ OrcProto.BucketStatistics.newBuilder().addCount(0)).setStringStatistics(
+ OrcProto.StringStatistics.newBuilder().setMaximum("zzz"));
}
// Ctor for memory estimation and tests
@@ -111,7 +104,7 @@ public final class OrcFileMetadata extends LlapCacheableBuffer implements FileMe
public OrcFileMetadata(long fileId, Reader reader) {
this.fileId = fileId;
this.stripeStats = reader.getOrcProtoStripeStatistics();
- this.compressionKind = reader.getCompression();
+ this.compressionKind = reader.getCompressionKind();
this.compressionBufferSize = reader.getCompressionSize();
this.stripes = reader.getStripes();
this.isOriginalFormat = OrcInputFormat.isOriginal(reader);
@@ -210,7 +203,7 @@ public final class OrcFileMetadata extends LlapCacheableBuffer implements FileMe
}
@Override
- public List<StripeStatistics> getStripeStats() {
+ public List<OrcProto.StripeStatistics> getStripeStats() {
return stripeStats;
}
@@ -225,7 +218,7 @@ public final class OrcFileMetadata extends LlapCacheableBuffer implements FileMe
}
@Override
- public List<ColumnStatistics> getFileStats() {
+ public List<OrcProto.ColumnStatistics> getFileStats() {
return fileStats;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
index d70ff1a..8479d22 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
@@ -26,26 +26,20 @@ import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator;
import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator;
import org.apache.hadoop.hive.llap.cache.EvictionDispatcher;
import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer;
-import org.apache.hadoop.hive.ql.io.orc.MetadataReader;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilter;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilterIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
-import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
+import org.apache.orc.impl.MetadataReader;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.StripeInformation;
import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey;
+import org.apache.orc.OrcProto;
import com.google.common.annotations.VisibleForTesting;
public class OrcStripeMetadata extends LlapCacheableBuffer {
private final OrcBatchKey stripeKey;
- private final List<ColumnEncoding> encodings;
- private final List<Stream> streams;
+ private final List<OrcProto.ColumnEncoding> encodings;
+ private final List<OrcProto.Stream> streams;
private final long rowCount;
- private RecordReaderImpl.Index rowIndex;
+ private OrcIndex rowIndex;
private final int estimatedMemUsage;
@@ -62,7 +56,7 @@ public class OrcStripeMetadata extends LlapCacheableBuffer {
public OrcStripeMetadata(OrcBatchKey stripeKey, MetadataReader mr, StripeInformation stripe,
boolean[] includes, boolean[] sargColumns) throws IOException {
this.stripeKey = stripeKey;
- StripeFooter footer = mr.readStripeFooter(stripe);
+ OrcProto.StripeFooter footer = mr.readStripeFooter(stripe);
streams = footer.getStreamsList();
encodings = footer.getColumnsList();
rowCount = stripe.getNumberOfRows();
@@ -81,15 +75,15 @@ public class OrcStripeMetadata extends LlapCacheableBuffer {
@VisibleForTesting
public static OrcStripeMetadata createDummy(long id) {
OrcStripeMetadata dummy = new OrcStripeMetadata(id);
- dummy.encodings.add(ColumnEncoding.getDefaultInstance());
- dummy.streams.add(Stream.getDefaultInstance());
- RowIndex ri = RowIndex.newBuilder().addEntry(
- RowIndexEntry.newBuilder().addPositions(1).setStatistics(
+ dummy.encodings.add(OrcProto.ColumnEncoding.getDefaultInstance());
+ dummy.streams.add(OrcProto.Stream.getDefaultInstance());
+ OrcProto.RowIndex ri = OrcProto.RowIndex.newBuilder().addEntry(
+ OrcProto.RowIndexEntry.newBuilder().addPositions(1).setStatistics(
OrcFileMetadata.createStatsDummy())).build();
- BloomFilterIndex bfi = BloomFilterIndex.newBuilder().addBloomFilter(
- BloomFilter.newBuilder().addBitset(0)).build();
- dummy.rowIndex = new RecordReaderImpl.Index(
- new RowIndex[] { ri }, new BloomFilterIndex[] { bfi });
+ OrcProto.BloomFilterIndex bfi = OrcProto.BloomFilterIndex.newBuilder().addBloomFilter(
+ OrcProto.BloomFilter.newBuilder().addBitset(0)).build();
+ dummy.rowIndex = new OrcIndex(
+ new OrcProto.RowIndex[] { ri }, new OrcProto.BloomFilterIndex[] { bfi });
return dummy;
}
@@ -112,19 +106,19 @@ public class OrcStripeMetadata extends LlapCacheableBuffer {
return stripeKey.stripeIx;
}
- public RowIndex[] getRowIndexes() {
+ public OrcProto.RowIndex[] getRowIndexes() {
return rowIndex.getRowGroupIndex();
}
- public BloomFilterIndex[] getBloomFilterIndexes() {
+ public OrcProto.BloomFilterIndex[] getBloomFilterIndexes() {
return rowIndex.getBloomFilterIndex();
}
- public List<ColumnEncoding> getEncodings() {
+ public List<OrcProto.ColumnEncoding> getEncodings() {
return encodings;
}
- public List<Stream> getStreams() {
+ public List<OrcProto.Stream> getStreams() {
return streams;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
index 7be82c2..a078f73 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
@@ -32,29 +32,12 @@ import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator;
import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator;
import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
-import org.apache.hadoop.hive.ql.io.orc.MetadataReader;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.BinaryStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilter;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilterIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding.Kind;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.DateStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.DecimalStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.TimestampStatistics;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.Index;
-import org.apache.hadoop.hive.ql.io.orc.StripeInformation;
+import org.apache.orc.impl.MetadataReader;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.StripeInformation;
import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
+import org.apache.orc.OrcProto;
import org.junit.Test;
import org.mockito.Mockito;
@@ -68,66 +51,68 @@ public class TestIncrementalObjectSizeEstimator {
public boolean isEmpty;
@Override
- public Index readRowIndex(StripeInformation stripe, StripeFooter footer,
- boolean[] included, RowIndex[] indexes, boolean[] sargColumns,
- BloomFilterIndex[] bloomFilterIndices) throws IOException {
+ public OrcIndex readRowIndex(StripeInformation stripe,
+ OrcProto.StripeFooter footer,
+ boolean[] included, OrcProto.RowIndex[] indexes, boolean[] sargColumns,
+ OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException {
if (isEmpty) {
- return new RecordReaderImpl.Index(new RowIndex[] { }, new BloomFilterIndex[] { });
+ return new OrcIndex(new OrcProto.RowIndex[] { },
+ new OrcProto.BloomFilterIndex[] { });
}
- ColumnStatistics cs = ColumnStatistics.newBuilder()
- .setBucketStatistics(BucketStatistics.newBuilder().addCount(0))
- .setStringStatistics(StringStatistics.newBuilder().setMaximum("zzz").setMinimum("aaa"))
- .setBinaryStatistics(BinaryStatistics.newBuilder().setSum(5))
- .setDateStatistics(DateStatistics.newBuilder().setMinimum(4545).setMaximum(6656))
- .setDecimalStatistics(DecimalStatistics.newBuilder().setMaximum("zzz").setMinimum("aaa"))
- .setDoubleStatistics(DoubleStatistics.newBuilder().setMinimum(0.5).setMaximum(1.5))
- .setIntStatistics(IntegerStatistics.newBuilder().setMaximum(10).setMinimum(5))
- .setTimestampStatistics(TimestampStatistics.newBuilder().setMaximum(10)).build();
- RowIndex ri = RowIndex.newBuilder()
- .addEntry(RowIndexEntry.newBuilder().addPositions(1))
- .addEntry(RowIndexEntry.newBuilder().addPositions(0).addPositions(2).setStatistics(cs))
+ OrcProto.ColumnStatistics cs = OrcProto.ColumnStatistics.newBuilder()
+ .setBucketStatistics(OrcProto.BucketStatistics.newBuilder().addCount(0))
+ .setStringStatistics(OrcProto.StringStatistics.newBuilder().setMaximum("zzz").setMinimum("aaa"))
+ .setBinaryStatistics(OrcProto.BinaryStatistics.newBuilder().setSum(5))
+ .setDateStatistics(OrcProto.DateStatistics.newBuilder().setMinimum(4545).setMaximum(6656))
+ .setDecimalStatistics(OrcProto.DecimalStatistics.newBuilder().setMaximum("zzz").setMinimum("aaa"))
+ .setDoubleStatistics(OrcProto.DoubleStatistics.newBuilder().setMinimum(0.5).setMaximum(1.5))
+ .setIntStatistics(OrcProto.IntegerStatistics.newBuilder().setMaximum(10).setMinimum(5))
+ .setTimestampStatistics(OrcProto.TimestampStatistics.newBuilder().setMaximum(10)).build();
+ OrcProto.RowIndex ri = OrcProto.RowIndex.newBuilder()
+ .addEntry(OrcProto.RowIndexEntry.newBuilder().addPositions(1))
+ .addEntry(OrcProto.RowIndexEntry.newBuilder().addPositions(0).addPositions(2).setStatistics(cs))
.build();
- RowIndex ri2 = RowIndex.newBuilder()
- .addEntry(RowIndexEntry.newBuilder().addPositions(3))
+ OrcProto.RowIndex ri2 = OrcProto.RowIndex.newBuilder()
+ .addEntry(OrcProto.RowIndexEntry.newBuilder().addPositions(3))
.build();
- BloomFilterIndex bfi = BloomFilterIndex.newBuilder().addBloomFilter(
- BloomFilter.newBuilder().addBitset(0).addBitset(1)).build();
+ OrcProto.BloomFilterIndex bfi = OrcProto.BloomFilterIndex.newBuilder().addBloomFilter(
+ OrcProto.BloomFilter.newBuilder().addBitset(0).addBitset(1)).build();
if (doStreamStep) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CodedOutputStream cos = CodedOutputStream.newInstance(baos);
ri.writeTo(cos);
cos.flush();
- ri = RowIndex.newBuilder().mergeFrom(baos.toByteArray()).build();
+ ri = OrcProto.RowIndex.newBuilder().mergeFrom(baos.toByteArray()).build();
baos = new ByteArrayOutputStream();
cos = CodedOutputStream.newInstance(baos);
ri2.writeTo(cos);
cos.flush();
- ri2 = RowIndex.newBuilder().mergeFrom(baos.toByteArray()).build();
+ ri2 = OrcProto.RowIndex.newBuilder().mergeFrom(baos.toByteArray()).build();
baos = new ByteArrayOutputStream();
cos = CodedOutputStream.newInstance(baos);
bfi.writeTo(cos);
cos.flush();
- bfi = BloomFilterIndex.newBuilder().mergeFrom(baos.toByteArray()).build();
+ bfi = OrcProto.BloomFilterIndex.newBuilder().mergeFrom(baos.toByteArray()).build();
}
- return new RecordReaderImpl.Index(
- new RowIndex[] { ri, ri2 }, new BloomFilterIndex[] { bfi });
+ return new OrcIndex(
+ new OrcProto.RowIndex[] { ri, ri2 }, new OrcProto.BloomFilterIndex[] { bfi });
}
@Override
- public StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
- StripeFooter.Builder fb = StripeFooter.newBuilder();
+ public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
+ OrcProto.StripeFooter.Builder fb = OrcProto.StripeFooter.newBuilder();
if (!isEmpty) {
- fb.addStreams(Stream.newBuilder().setColumn(0).setLength(20).setKind(Stream.Kind.LENGTH))
- .addStreams(Stream.newBuilder().setColumn(0).setLength(40).setKind(Stream.Kind.DATA))
- .addColumns(ColumnEncoding.newBuilder().setDictionarySize(10).setKind(Kind.DIRECT_V2));
+ fb.addStreams(OrcProto.Stream.newBuilder().setColumn(0).setLength(20).setKind(OrcProto.Stream.Kind.LENGTH))
+ .addStreams(OrcProto.Stream.newBuilder().setColumn(0).setLength(40).setKind(OrcProto.Stream.Kind.DATA))
+ .addColumns(OrcProto.ColumnEncoding.newBuilder().setDictionarySize(10).setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2));
}
- StripeFooter footer = fb.build();
+ OrcProto.StripeFooter footer = fb.build();
if (doStreamStep) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CodedOutputStream cos = CodedOutputStream.newInstance(baos);
footer.writeTo(cos);
cos.flush();
- footer = StripeFooter.newBuilder().mergeFrom(baos.toByteArray()).build();
+ footer = OrcProto.StripeFooter.newBuilder().mergeFrom(baos.toByteArray()).build();
}
return footer;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c7a78ee/orc/pom.xml
----------------------------------------------------------------------
diff --git a/orc/pom.xml b/orc/pom.xml
new file mode 100644
index 0000000..fef23e8
--- /dev/null
+++ b/orc/pom.xml
@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>hive-orc</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive ORC</name>
+
+ <properties>
+ <hive.path.to.root>..</hive.path.to.root>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-storage-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- inter-project -->
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.iq80.snappy</groupId>
+ <artifactId>snappy</artifactId>
+ <version>${snappy.version}</version>
+ </dependency>
+
+ <!-- test inter-project -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito-all.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>protobuf</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.github.os72</groupId>
+ <artifactId>protoc-jar-maven-plugin</artifactId>
+ <version>3.0.0-a3</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <protocVersion>2.5.0</protocVersion>
+ <addSources>none</addSources>
+ <outputDirectory>src/gen/protobuf-java</outputDirectory>
+ <includeDirectories>
+ <include>src/protobuf</include>
+ </includeDirectories>
+ <inputDirectories>
+ <include>src/protobuf</include>
+ </inputDirectories>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ <sourceDirectory>${basedir}/src/java</sourceDirectory>
+ <testSourceDirectory>${basedir}/src/test</testSourceDirectory>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/gen/protobuf-java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>