You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2016/05/20 21:22:46 UTC
[08/27] hive git commit: HIVE-11417. Move the ReaderImpl and
RowReaderImpl to the ORC module,
by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
deleted file mode 100644
index f41a7ba..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
+++ /dev/null
@@ -1,1342 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-
-import java.io.File;
-import java.sql.Timestamp;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.orc.CompressionKind;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Longs;
-
-@RunWith(value = Parameterized.class)
-public class TestNewIntegerEncoding {
-
- private OrcFile.EncodingStrategy encodingStrategy;
-
- public TestNewIntegerEncoding( OrcFile.EncodingStrategy es) {
- this.encodingStrategy = es;
- }
-
- @Parameters
- public static Collection<Object[]> data() {
- Object[][] data = new Object[][] { { OrcFile.EncodingStrategy.COMPRESSION },
- { OrcFile.EncodingStrategy.SPEED } };
- return Arrays.asList(data);
- }
-
- public static class TSRow {
- Timestamp ts;
-
- public TSRow(Timestamp ts) {
- this.ts = ts;
- }
- }
-
- public static class Row {
- Integer int1;
- Long long1;
-
- public Row(int val, long l) {
- this.int1 = val;
- this.long1 = l;
- }
- }
-
- Path workDir = new Path(System.getProperty("test.tmp.dir", "target"
- + File.separator + "test" + File.separator + "tmp"));
-
- Configuration conf;
- FileSystem fs;
- Path testFilePath;
-
- @Rule
- public TestName testCaseName = new TestName();
-
- @Before
- public void openFileSystem() throws Exception {
- conf = new Configuration();
- fs = FileSystem.getLocal(conf);
- testFilePath = new Path(workDir, "TestOrcFile."
- + testCaseName.getMethodName() + ".orc");
- fs.delete(testFilePath, false);
- }
-
- @Test
- public void testBasicRow() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Row.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- writer.addRow(new Row(111, 1111L));
- writer.addRow(new Row(111, 1111L));
- writer.addRow(new Row(111, 1111L));
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(new IntWritable(111), ((OrcStruct) row).getFieldValue(0));
- assertEquals(new LongWritable(1111), ((OrcStruct) row).getFieldValue(1));
- }
- }
-
- @Test
- public void testBasicOld() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 6,
- 7, 8, 9, 10, 1, 1, 1, 1, 1, 1, 10, 9, 7, 6, 5, 4, 3, 2, 1, 1, 1, 1, 1,
- 2, 5, 1, 3, 7, 1, 9, 2, 6, 3, 7, 1, 9, 2, 6, 3, 7, 1, 9, 2, 6, 3, 7, 1,
- 9, 2, 6, 3, 7, 1, 9, 2, 6, 2000, 2, 1, 1, 1, 1, 1, 3, 7, 1, 9, 2, 6, 1,
- 1, 1, 1, 1 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .compress(CompressionKind.NONE)
- .version(OrcFile.Version.V_0_11)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testBasicNew() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 6,
- 7, 8, 9, 10, 1, 1, 1, 1, 1, 1, 10, 9, 7, 6, 5, 4, 3, 2, 1, 1, 1, 1, 1,
- 2, 5, 1, 3, 7, 1, 9, 2, 6, 3, 7, 1, 9, 2, 6, 3, 7, 1, 9, 2, 6, 3, 7, 1,
- 9, 2, 6, 3, 7, 1, 9, 2, 6, 2000, 2, 1, 1, 1, 1, 1, 3, 7, 1, 9, 2, 6, 1,
- 1, 1, 1, 1 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testBasicDelta1() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { -500, -400, -350, -325, -310 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testBasicDelta2() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { -500, -600, -650, -675, -710 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testBasicDelta3() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { 500, 400, 350, 325, 310 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testBasicDelta4() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { 500, 600, 650, 675, 710 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testDeltaOverflow() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory
- .getReflectionObjectInspector(Long.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[]{4513343538618202719l, 4513343538618202711l,
- 2911390882471569739l,
- -9181829309989854913l};
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(
- testFilePath,
- OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000)
- .compress(CompressionKind.NONE).bufferSize(10000));
- for (Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile
- .createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testDeltaOverflow2() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory
- .getReflectionObjectInspector(Long.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[]{Long.MAX_VALUE, 4513343538618202711l,
- 2911390882471569739l,
- Long.MIN_VALUE};
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(
- testFilePath,
- OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000)
- .compress(CompressionKind.NONE).bufferSize(10000));
- for (Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile
- .createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testDeltaOverflow3() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory
- .getReflectionObjectInspector(Long.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[]{-4513343538618202711l, -2911390882471569739l, -2,
- Long.MAX_VALUE};
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(
- testFilePath,
- OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000)
- .compress(CompressionKind.NONE).bufferSize(10000));
- for (Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile
- .createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testIntegerMin() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- input.add((long) Integer.MIN_VALUE);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testIntegerMax() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- input.add((long) Integer.MAX_VALUE);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testLongMin() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- input.add(Long.MIN_VALUE);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testLongMax() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- input.add(Long.MAX_VALUE);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testRandomInt() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for(int i = 0; i < 100000; i++) {
- input.add((long) rand.nextInt());
- }
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testRandomLong() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for(int i = 0; i < 100000; i++) {
- input.add(rand.nextLong());
- }
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseNegativeMin() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { 20, 2, 3, 2, 1, 3, 17, 71, 35, 2, 1, 139, 2, 2,
- 3, 1783, 475, 2, 1, 1, 3, 1, 3, 2, 32, 1, 2, 3, 1, 8, 30, 1, 3, 414, 1,
- 1, 135, 3, 3, 1, 414, 2, 1, 2, 2, 594, 2, 5, 6, 4, 11, 1, 2, 2, 1, 1,
- 52, 4, 1, 2, 7, 1, 17, 334, 1, 2, 1, 2, 2, 6, 1, 266, 1, 2, 217, 2, 6,
- 2, 13, 2, 2, 1, 2, 3, 5, 1, 2, 1, 7244, 11813, 1, 33, 2, -13, 1, 2, 3,
- 13, 1, 92, 3, 13, 5, 14, 9, 141, 12, 6, 15, 25, 1, 1, 1, 46, 2, 1, 1,
- 141, 3, 1, 1, 1, 1, 2, 1, 4, 34, 5, 78, 8, 1, 2, 2, 1, 9, 10, 2, 1, 4,
- 13, 1, 5, 4, 4, 19, 5, 1, 1, 1, 68, 33, 399, 1, 1885, 25, 5, 2, 4, 1,
- 1, 2, 16, 1, 2966, 3, 1, 1, 25501, 1, 1, 1, 66, 1, 3, 8, 131, 14, 5, 1,
- 2, 2, 1, 1, 8, 1, 1, 2, 1, 5, 9, 2, 3, 112, 13, 2, 2, 1, 5, 10, 3, 1,
- 1, 13, 2, 3, 4, 1, 3, 1, 1, 2, 1, 1, 2, 4, 2, 207, 1, 1, 2, 4, 3, 3, 2,
- 2, 16 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseNegativeMin2() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { 20, 2, 3, 2, 1, 3, 17, 71, 35, 2, 1, 139, 2, 2,
- 3, 1783, 475, 2, 1, 1, 3, 1, 3, 2, 32, 1, 2, 3, 1, 8, 30, 1, 3, 414, 1,
- 1, 135, 3, 3, 1, 414, 2, 1, 2, 2, 594, 2, 5, 6, 4, 11, 1, 2, 2, 1, 1,
- 52, 4, 1, 2, 7, 1, 17, 334, 1, 2, 1, 2, 2, 6, 1, 266, 1, 2, 217, 2, 6,
- 2, 13, 2, 2, 1, 2, 3, 5, 1, 2, 1, 7244, 11813, 1, 33, 2, -1, 1, 2, 3,
- 13, 1, 92, 3, 13, 5, 14, 9, 141, 12, 6, 15, 25, 1, 1, 1, 46, 2, 1, 1,
- 141, 3, 1, 1, 1, 1, 2, 1, 4, 34, 5, 78, 8, 1, 2, 2, 1, 9, 10, 2, 1, 4,
- 13, 1, 5, 4, 4, 19, 5, 1, 1, 1, 68, 33, 399, 1, 1885, 25, 5, 2, 4, 1,
- 1, 2, 16, 1, 2966, 3, 1, 1, 25501, 1, 1, 1, 66, 1, 3, 8, 131, 14, 5, 1,
- 2, 2, 1, 1, 8, 1, 1, 2, 1, 5, 9, 2, 3, 112, 13, 2, 2, 1, 5, 10, 3, 1,
- 1, 13, 2, 3, 4, 1, 3, 1, 1, 2, 1, 1, 2, 4, 2, 207, 1, 1, 2, 4, 3, 3, 2,
- 2, 16 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseNegativeMin3() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { 20, 2, 3, 2, 1, 3, 17, 71, 35, 2, 1, 139, 2, 2,
- 3, 1783, 475, 2, 1, 1, 3, 1, 3, 2, 32, 1, 2, 3, 1, 8, 30, 1, 3, 414, 1,
- 1, 135, 3, 3, 1, 414, 2, 1, 2, 2, 594, 2, 5, 6, 4, 11, 1, 2, 2, 1, 1,
- 52, 4, 1, 2, 7, 1, 17, 334, 1, 2, 1, 2, 2, 6, 1, 266, 1, 2, 217, 2, 6,
- 2, 13, 2, 2, 1, 2, 3, 5, 1, 2, 1, 7244, 11813, 1, 33, 2, 0, 1, 2, 3,
- 13, 1, 92, 3, 13, 5, 14, 9, 141, 12, 6, 15, 25, 1, 1, 1, 46, 2, 1, 1,
- 141, 3, 1, 1, 1, 1, 2, 1, 4, 34, 5, 78, 8, 1, 2, 2, 1, 9, 10, 2, 1, 4,
- 13, 1, 5, 4, 4, 19, 5, 1, 1, 1, 68, 33, 399, 1, 1885, 25, 5, 2, 4, 1,
- 1, 2, 16, 1, 2966, 3, 1, 1, 25501, 1, 1, 1, 66, 1, 3, 8, 131, 14, 5, 1,
- 2, 2, 1, 1, 8, 1, 1, 2, 1, 5, 9, 2, 3, 112, 13, 2, 2, 1, 5, 10, 3, 1,
- 1, 13, 2, 3, 4, 1, 3, 1, 1, 2, 1, 1, 2, 4, 2, 207, 1, 1, 2, 4, 3, 3, 2,
- 2, 16 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseNegativeMin4() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- long[] inp = new long[] { 13, 13, 11, 8, 13, 10, 10, 11, 11, 14, 11, 7, 13,
- 12, 12, 11, 15, 12, 12, 9, 8, 10, 13, 11, 8, 6, 5, 6, 11, 7, 15, 10, 7,
- 6, 8, 7, 9, 9, 11, 33, 11, 3, 7, 4, 6, 10, 14, 12, 5, 14, 7, 6 };
- List<Long> input = Lists.newArrayList(Longs.asList(inp));
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseAt0() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for(int i = 0; i < 5120; i++) {
- input.add((long) rand.nextInt(100));
- }
- input.set(0, 20000L);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseAt1() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for(int i = 0; i < 5120; i++) {
- input.add((long) rand.nextInt(100));
- }
- input.set(1, 20000L);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseAt255() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for(int i = 0; i < 5120; i++) {
- input.add((long) rand.nextInt(100));
- }
- input.set(255, 20000L);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseAt256() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for(int i = 0; i < 5120; i++) {
- input.add((long) rand.nextInt(100));
- }
- input.set(256, 20000L);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBase510() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for(int i = 0; i < 5120; i++) {
- input.add((long) rand.nextInt(100));
- }
- input.set(510, 20000L);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBase511() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for(int i = 0; i < 5120; i++) {
- input.add((long) rand.nextInt(100));
- }
- input.set(511, 20000L);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseMax1() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for (int i = 0; i < 5120; i++) {
- input.add((long) rand.nextInt(60));
- }
- input.set(511, Long.MAX_VALUE);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for (Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseMax2() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for (int i = 0; i < 5120; i++) {
- input.add((long) rand.nextInt(60));
- }
- input.set(128, Long.MAX_VALUE);
- input.set(256, Long.MAX_VALUE);
- input.set(511, Long.MAX_VALUE);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for (Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseMax3() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- input.add(371946367L);
- input.add(11963367L);
- input.add(68639400007L);
- input.add(100233367L);
- input.add(6367L);
- input.add(10026367L);
- input.add(3670000L);
- input.add(3602367L);
- input.add(4719226367L);
- input.add(7196367L);
- input.add(444442L);
- input.add(210267L);
- input.add(21033L);
- input.add(160267L);
- input.add(400267L);
- input.add(23634347L);
- input.add(16027L);
- input.add(46026367L);
- input.add(Long.MAX_VALUE);
- input.add(33333L);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for (Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseMax4() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- for (int i = 0; i < 25; i++) {
- input.add(371292224226367L);
- input.add(119622332222267L);
- input.add(686329400222007L);
- input.add(100233333222367L);
- input.add(636272333322222L);
- input.add(10202633223267L);
- input.add(36700222022230L);
- input.add(36023226224227L);
- input.add(47192226364427L);
- input.add(71963622222447L);
- input.add(22244444222222L);
- input.add(21220263327442L);
- input.add(21032233332232L);
- input.add(16026322232227L);
- input.add(40022262272212L);
- input.add(23634342227222L);
- input.add(16022222222227L);
- input.add(46026362222227L);
- input.add(46026362222227L);
- input.add(33322222222323L);
- }
- input.add(Long.MAX_VALUE);
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
- for (Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-
- @Test
- public void testPatchedBaseTimestamp() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(TSRow.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
-
- List<Timestamp> tslist = Lists.newArrayList();
- tslist.add(Timestamp.valueOf("2099-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2003-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("1999-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("1995-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2002-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2010-03-02 00:00:00"));
- tslist.add(Timestamp.valueOf("2005-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2006-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2003-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("1996-08-02 00:00:00"));
- tslist.add(Timestamp.valueOf("1998-11-02 00:00:00"));
- tslist.add(Timestamp.valueOf("2008-10-02 00:00:00"));
- tslist.add(Timestamp.valueOf("1993-08-02 00:00:00"));
- tslist.add(Timestamp.valueOf("2008-01-02 00:00:00"));
- tslist.add(Timestamp.valueOf("2007-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2004-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2008-10-02 00:00:00"));
- tslist.add(Timestamp.valueOf("2003-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2004-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2008-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2005-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("1994-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2006-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2004-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2001-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2000-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2000-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2002-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2006-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2011-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2002-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("2005-01-01 00:00:00"));
- tslist.add(Timestamp.valueOf("1974-01-01 00:00:00"));
-
- for (Timestamp ts : tslist) {
- writer.addRow(new TSRow(ts));
- }
-
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(tslist.get(idx++).getNanos(),
- ((TimestampWritable) ((OrcStruct) row).getFieldValue(0)).getNanos());
- }
- }
-
- @Test
- public void testDirectLargeNegatives() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000)
- .encodingStrategy(encodingStrategy));
-
- writer.addRow(-7486502418706614742L);
- writer.addRow(0L);
- writer.addRow(1L);
- writer.addRow(1L);
- writer.addRow(-5535739865598783616L);
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- Object row = rows.next(null);
- assertEquals(-7486502418706614742L, ((LongWritable) row).get());
- row = rows.next(row);
- assertEquals(0L, ((LongWritable) row).get());
- row = rows.next(row);
- assertEquals(1L, ((LongWritable) row).get());
- row = rows.next(row);
- assertEquals(1L, ((LongWritable) row).get());
- row = rows.next(row);
- assertEquals(-5535739865598783616L, ((LongWritable) row).get());
- }
-
- @Test
- public void testSeek() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- List<Long> input = Lists.newArrayList();
- Random rand = new Random();
- for(int i = 0; i < 100000; i++) {
- input.add((long) rand.nextInt());
- }
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .compress(CompressionKind.NONE)
- .stripeSize(100000)
- .bufferSize(10000)
- .version(OrcFile.Version.V_0_11)
- .encodingStrategy(encodingStrategy));
- for(Long l : input) {
- writer.addRow(l);
- }
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- int idx = 55555;
- rows.seekToRow(idx);
- while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
index 1a97a6d..c7c2c9d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.TypeDesc;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
@@ -537,7 +538,7 @@ public class TestOrcFile {
Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -574,7 +575,7 @@ public class TestOrcFile {
List<? extends StructField> fields = readerInspector.getAllStructFieldRefs();
HiveDecimalObjectInspector doi = (HiveDecimalObjectInspector) readerInspector.
getStructFieldRef("dec").getFieldObjectInspector();
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
while (rows.hasNext()) {
Object row = rows.next(null);
assertEquals(null, doi.getPrimitiveWritableObject(readerInspector.getStructFieldData(row,
@@ -617,7 +618,7 @@ public class TestOrcFile {
List<? extends StructField> fields = readerInspector.getAllStructFieldRefs();
HiveDecimalObjectInspector doi = (HiveDecimalObjectInspector) readerInspector.
getStructFieldRef("dec").getFieldObjectInspector();
- RecordReader rows = reader.rows(null);
+ RecordReader rows = reader.rows();
int idx = 0;
while (rows.hasNext()) {
Object row = rows.next(null);
@@ -1702,6 +1703,11 @@ public class TestOrcFile {
RecordReader rows = reader.rows();
OrcStruct row = null;
for(int i=COUNT-1; i >= 0; --i) {
+ // since we are walking backwards, seek back a buffer width so that
+ // we load the previous buffer of rows
+ if (i % COUNT == COUNT - 1) {
+ rows.seekToRow(i - (COUNT - 1));
+ }
rows.seekToRow(i);
row = (OrcStruct) rows.next(row);
BigRow expected = createRandomRow(intValues, doubleValues,
@@ -1816,6 +1822,11 @@ public class TestOrcFile {
/* all tests are identical to the other seek() tests */
OrcStruct row = null;
for(int i=COUNT-1; i >= 0; --i) {
+ // since we are walking backwards, seek back a buffer width so that
+ // we load the previous buffer of rows
+ if (i % COUNT == COUNT - 1) {
+ rows.seekToRow(i - (COUNT - 1));
+ }
rows.seekToRow(i);
row = (OrcStruct) rows.next(row);
BigRow expected = createRandomRow(intValues, doubleValues,
@@ -2067,10 +2078,11 @@ public class TestOrcFile {
.range(0L, Long.MAX_VALUE)
.include(new boolean[]{true, true, true})
.searchArgument(sarg, new String[]{null, "int1", "string1"}));
- assertEquals(1000L, rows.getRowNumber());
+ assertEquals(0L, rows.getRowNumber());
OrcStruct row = null;
for(int i=1000; i < 2000; ++i) {
assertTrue(rows.hasNext());
+ assertEquals(i, rows.getRowNumber());
row = (OrcStruct) rows.next(row);
assertEquals(300 * i, ((IntWritable) row.getFieldValue(0)).get());
assertEquals(Integer.toHexString(10*i), row.getFieldValue(1).toString());
@@ -2088,7 +2100,6 @@ public class TestOrcFile {
.range(0L, Long.MAX_VALUE)
.include(new boolean[]{true, true, true})
.searchArgument(sarg, new String[]{null, "int1", "string1"}));
- assertEquals(3500L, rows.getRowNumber());
assertTrue(!rows.hasNext());
// select first 100 and last 100 rows
@@ -2154,4 +2165,53 @@ public class TestOrcFile {
Assert.assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
}
}
+
+ static class MyList {
+ List<Integer> list = new ArrayList<>();
+ }
+
+ @Test
+ public void testListExpansion() throws Exception {
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector
+ (MyList.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ Writer writer = OrcFile.createWriter(testFilePath,
+ OrcFile.writerOptions(conf).inspector(inspector));
+ MyList row = new MyList();
+ row.list.add(1);
+ row.list.add(2);
+ row.list.add(3);
+ writer.addRow(row);
+ row.list.clear();
+ writer.addRow(row);
+ row.list.add(11);
+ row.list.add(12);
+ writer.addRow(row);
+ row.list = null;
+ writer.addRow(row);
+ row.list = new ArrayList<>();
+ row.list.add(21);
+ row.list.add(22);
+ row.list.add(23);
+ row.list.add(24);
+ writer.addRow(row);
+ writer.close();
+ RecordReader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf)).rows();
+ assertEquals(true, reader.hasNext());
+ OrcStruct orcrow = (OrcStruct) reader.next(null);
+ assertEquals(3, ((List<IntWritable>) orcrow.getFieldValue(0)).size());
+ orcrow = (OrcStruct) reader.next(row);
+ assertEquals(0, ((List<IntWritable>) orcrow.getFieldValue(0)).size());
+ orcrow = (OrcStruct) reader.next(row);
+ assertEquals(2, ((List<IntWritable>) orcrow.getFieldValue(0)).size());
+ assertEquals(null, ((OrcStruct) reader.next(row)).getFieldValue(0));
+ orcrow = (OrcStruct) reader.next(row);
+ assertEquals(4, ((List<IntWritable>) orcrow.getFieldValue(0)).size());
+ assertEquals(false, reader.hasNext());
+ reader.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
deleted file mode 100644
index e96c809..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.orc.ColumnStatistics;
-import org.apache.orc.CompressionKind;
-import org.apache.orc.IntegerColumnStatistics;
-import org.apache.orc.OrcProto;
-
-import org.apache.orc.StringColumnStatistics;
-import org.apache.orc.StripeInformation;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-import com.google.common.collect.Lists;
-
-public class TestOrcNullOptimization {
-
- public static class MyStruct {
- Integer a;
- String b;
- Boolean c;
- List<InnerStruct> list = new ArrayList<InnerStruct>();
-
- public MyStruct(Integer a, String b, Boolean c, List<InnerStruct> l) {
- this.a = a;
- this.b = b;
- this.c = c;
- this.list = l;
- }
- }
-
- public static class InnerStruct {
- Integer z;
-
- public InnerStruct(int z) {
- this.z = z;
- }
- }
-
- Path workDir = new Path(System.getProperty("test.tmp.dir",
- "target" + File.separator + "test" + File.separator + "tmp"));
-
- Configuration conf;
- FileSystem fs;
- Path testFilePath;
-
- @Rule
- public TestName testCaseName = new TestName();
-
- @Before
- public void openFileSystem() throws Exception {
- conf = new Configuration();
- fs = FileSystem.getLocal(conf);
- testFilePath = new Path(workDir, "TestOrcNullOptimization." +
- testCaseName.getMethodName() + ".orc");
- fs.delete(testFilePath, false);
- }
-
- @Test
- public void testMultiStripeWithNull() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcNullOptimization.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector
- (MyStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000));
- Random rand = new Random(100);
- writer.addRow(new MyStruct(null, null, true,
- Lists.newArrayList(new InnerStruct(100))));
- for (int i = 2; i < 20000; i++) {
- writer.addRow(new MyStruct(rand.nextInt(1), "a", true, Lists
- .newArrayList(new InnerStruct(100))));
- }
- writer.addRow(new MyStruct(null, null, true,
- Lists.newArrayList(new InnerStruct(100))));
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- // check the stats
- ColumnStatistics[] stats = reader.getStatistics();
- assertEquals(20000, reader.getNumberOfRows());
- assertEquals(20000, stats[0].getNumberOfValues());
-
- assertEquals(0, ((IntegerColumnStatistics) stats[1]).getMaximum());
- assertEquals(0, ((IntegerColumnStatistics) stats[1]).getMinimum());
- assertEquals(true, ((IntegerColumnStatistics) stats[1]).isSumDefined());
- assertEquals(0, ((IntegerColumnStatistics) stats[1]).getSum());
- assertEquals("count: 19998 hasNull: true min: 0 max: 0 sum: 0",
- stats[1].toString());
-
- assertEquals("a", ((StringColumnStatistics) stats[2]).getMaximum());
- assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum());
- assertEquals(19998, stats[2].getNumberOfValues());
- assertEquals("count: 19998 hasNull: true min: a max: a sum: 19998",
- stats[2].toString());
-
- // check the inspectors
- StructObjectInspector readerInspector =
- (StructObjectInspector) reader.getObjectInspector();
- assertEquals(ObjectInspector.Category.STRUCT,
- readerInspector.getCategory());
- assertEquals("struct<a:int,b:string,c:boolean,list:array<struct<z:int>>>",
- readerInspector.getTypeName());
-
- RecordReader rows = reader.rows();
-
- List<Boolean> expected = Lists.newArrayList();
- for (StripeInformation sinfo : reader.getStripes()) {
- expected.add(false);
- }
- // only the first and last stripe will have PRESENT stream
- expected.set(0, true);
- expected.set(expected.size() - 1, true);
-
- List<Boolean> got = Lists.newArrayList();
- // check if the strip footer contains PRESENT stream
- for (StripeInformation sinfo : reader.getStripes()) {
- OrcProto.StripeFooter sf =
- ((RecordReaderImpl) rows).readStripeFooter(sinfo);
- got.add(sf.toString().indexOf(OrcProto.Stream.Kind.PRESENT.toString())
- != -1);
- }
- assertEquals(expected, got);
-
- // row 1
- OrcStruct row = (OrcStruct) rows.next(null);
- assertNotNull(row);
- assertNull(row.getFieldValue(0));
- assertNull(row.getFieldValue(1));
- assertEquals(new BooleanWritable(true), row.getFieldValue(2));
- assertEquals(new IntWritable(100),
- ((OrcStruct) ((ArrayList<?>) row.getFieldValue(3)).get(0)).
- getFieldValue(0));
-
- rows.seekToRow(19998);
- // last-1 row
- row = (OrcStruct) rows.next(null);
- assertNotNull(row);
- assertNotNull(row.getFieldValue(1));
- assertEquals(new IntWritable(0), row.getFieldValue(0));
- assertEquals(new BooleanWritable(true), row.getFieldValue(2));
- assertEquals(new IntWritable(100),
- ((OrcStruct) ((ArrayList<?>) row.getFieldValue(3)).get(0)).
- getFieldValue(0));
-
- // last row
- row = (OrcStruct) rows.next(row);
- assertNotNull(row);
- assertNull(row.getFieldValue(0));
- assertNull(row.getFieldValue(1));
- assertEquals(new BooleanWritable(true), row.getFieldValue(2));
- assertEquals(new IntWritable(100),
- ((OrcStruct) ((ArrayList<?>) row.getFieldValue(3)).get(0)).
- getFieldValue(0));
-
- rows.close();
- }
-
- @Test
- public void testMultiStripeWithoutNull() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcNullOptimization.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector
- (MyStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .compress(CompressionKind.NONE)
- .bufferSize(10000));
- Random rand = new Random(100);
- for (int i = 1; i < 20000; i++) {
- writer.addRow(new MyStruct(rand.nextInt(1), "a", true, Lists
- .newArrayList(new InnerStruct(100))));
- }
- writer.addRow(new MyStruct(0, "b", true,
- Lists.newArrayList(new InnerStruct(100))));
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- // check the stats
- ColumnStatistics[] stats = reader.getStatistics();
- assertEquals(20000, reader.getNumberOfRows());
- assertEquals(20000, stats[0].getNumberOfValues());
-
- assertEquals(0, ((IntegerColumnStatistics) stats[1]).getMaximum());
- assertEquals(0, ((IntegerColumnStatistics) stats[1]).getMinimum());
- assertEquals(true, ((IntegerColumnStatistics) stats[1]).isSumDefined());
- assertEquals(0, ((IntegerColumnStatistics) stats[1]).getSum());
- assertEquals("count: 20000 hasNull: false min: 0 max: 0 sum: 0",
- stats[1].toString());
-
- assertEquals("b", ((StringColumnStatistics) stats[2]).getMaximum());
- assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum());
- assertEquals(20000, stats[2].getNumberOfValues());
- assertEquals("count: 20000 hasNull: false min: a max: b sum: 20000",
- stats[2].toString());
-
- // check the inspectors
- StructObjectInspector readerInspector =
- (StructObjectInspector) reader.getObjectInspector();
- assertEquals(ObjectInspector.Category.STRUCT,
- readerInspector.getCategory());
- assertEquals("struct<a:int,b:string,c:boolean,list:array<struct<z:int>>>",
- readerInspector.getTypeName());
-
- RecordReader rows = reader.rows();
-
- // none of the stripes will have PRESENT stream
- List<Boolean> expected = Lists.newArrayList();
- for (StripeInformation sinfo : reader.getStripes()) {
- expected.add(false);
- }
-
- List<Boolean> got = Lists.newArrayList();
- // check if the strip footer contains PRESENT stream
- for (StripeInformation sinfo : reader.getStripes()) {
- OrcProto.StripeFooter sf =
- ((RecordReaderImpl) rows).readStripeFooter(sinfo);
- got.add(sf.toString().indexOf(OrcProto.Stream.Kind.PRESENT.toString())
- != -1);
- }
- assertEquals(expected, got);
-
- rows.seekToRow(19998);
- // last-1 row
- OrcStruct row = (OrcStruct) rows.next(null);
- assertNotNull(row);
- assertNotNull(row.getFieldValue(1));
- assertEquals(new IntWritable(0), row.getFieldValue(0));
- assertEquals("a", row.getFieldValue(1).toString());
- assertEquals(new BooleanWritable(true), row.getFieldValue(2));
- assertEquals(new IntWritable(100),
- ((OrcStruct) ((ArrayList<?>) row.getFieldValue(3)).get(0)).
- getFieldValue(0));
-
- // last row
- row = (OrcStruct) rows.next(row);
- assertNotNull(row);
- assertNotNull(row.getFieldValue(0));
- assertNotNull(row.getFieldValue(1));
- assertEquals("b", row.getFieldValue(1).toString());
- assertEquals(new BooleanWritable(true), row.getFieldValue(2));
- assertEquals(new IntWritable(100),
- ((OrcStruct) ((ArrayList<?>) row.getFieldValue(3)).get(0)).
- getFieldValue(0));
- rows.close();
- }
-
- @Test
- public void testColumnsWithNullAndCompression() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcNullOptimization.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector
- (MyStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(100000)
- .bufferSize(10000));
- writer.addRow(new MyStruct(3, "a", true,
- Lists.newArrayList(new InnerStruct(100))));
- writer.addRow(new MyStruct(null, "b", true,
- Lists.newArrayList(new InnerStruct(100))));
- writer.addRow(new MyStruct(3, null, false,
- Lists.newArrayList(new InnerStruct(100))));
- writer.addRow(new MyStruct(3, "d", true,
- Lists.newArrayList(new InnerStruct(100))));
- writer.addRow(new MyStruct(2, "e", true,
- Lists.newArrayList(new InnerStruct(100))));
- writer.addRow(new MyStruct(2, "f", true,
- Lists.newArrayList(new InnerStruct(100))));
- writer.addRow(new MyStruct(2, "g", true,
- Lists.newArrayList(new InnerStruct(100))));
- writer.addRow(new MyStruct(2, "h", true,
- Lists.newArrayList(new InnerStruct(100))));
- writer.close();
-
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- // check the stats
- ColumnStatistics[] stats = reader.getStatistics();
- assertEquals(8, reader.getNumberOfRows());
- assertEquals(8, stats[0].getNumberOfValues());
-
- assertEquals(3, ((IntegerColumnStatistics) stats[1]).getMaximum());
- assertEquals(2, ((IntegerColumnStatistics) stats[1]).getMinimum());
- assertEquals(true, ((IntegerColumnStatistics) stats[1]).isSumDefined());
- assertEquals(17, ((IntegerColumnStatistics) stats[1]).getSum());
- assertEquals("count: 7 hasNull: true min: 2 max: 3 sum: 17",
- stats[1].toString());
-
- assertEquals("h", ((StringColumnStatistics) stats[2]).getMaximum());
- assertEquals("a", ((StringColumnStatistics) stats[2]).getMinimum());
- assertEquals(7, stats[2].getNumberOfValues());
- assertEquals("count: 7 hasNull: true min: a max: h sum: 7",
- stats[2].toString());
-
- // check the inspectors
- StructObjectInspector readerInspector =
- (StructObjectInspector) reader.getObjectInspector();
- assertEquals(ObjectInspector.Category.STRUCT,
- readerInspector.getCategory());
- assertEquals("struct<a:int,b:string,c:boolean,list:array<struct<z:int>>>",
- readerInspector.getTypeName());
-
- RecordReader rows = reader.rows();
- // only the last strip will have PRESENT stream
- List<Boolean> expected = Lists.newArrayList();
- for (StripeInformation sinfo : reader.getStripes()) {
- expected.add(false);
- }
- expected.set(expected.size() - 1, true);
-
- List<Boolean> got = Lists.newArrayList();
- // check if the strip footer contains PRESENT stream
- for (StripeInformation sinfo : reader.getStripes()) {
- OrcProto.StripeFooter sf =
- ((RecordReaderImpl) rows).readStripeFooter(sinfo);
- got.add(sf.toString().indexOf(OrcProto.Stream.Kind.PRESENT.toString())
- != -1);
- }
- assertEquals(expected, got);
-
- // row 1
- OrcStruct row = (OrcStruct) rows.next(null);
- assertNotNull(row);
- assertEquals(new IntWritable(3), row.getFieldValue(0));
- assertEquals("a", row.getFieldValue(1).toString());
- assertEquals(new BooleanWritable(true), row.getFieldValue(2));
- assertEquals(new IntWritable(100),
- ((OrcStruct) ((ArrayList<?>) row.getFieldValue(3)).get(0)).
- getFieldValue(0));
-
- // row 2
- row = (OrcStruct) rows.next(row);
- assertNotNull(row);
- assertNull(row.getFieldValue(0));
- assertEquals("b", row.getFieldValue(1).toString());
- assertEquals(new BooleanWritable(true), row.getFieldValue(2));
- assertEquals(new IntWritable(100),
- ((OrcStruct) ((ArrayList<?>) row.getFieldValue(3)).get(0)).
- getFieldValue(0));
-
- // row 3
- row = (OrcStruct) rows.next(row);
- assertNotNull(row);
- assertNull(row.getFieldValue(1));
- assertEquals(new IntWritable(3), row.getFieldValue(0));
- assertEquals(new BooleanWritable(false), row.getFieldValue(2));
- assertEquals(new IntWritable(100),
- ((OrcStruct) ((ArrayList<?>) row.getFieldValue(3)).get(0)).
- getFieldValue(0));
- rows.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
index 973cc40..0a61fb8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.orc.impl.OrcAcidUtils;
+import org.apache.orc.tools.FileDump;
import org.junit.Test;
public class TestOrcRecordUpdater {
@@ -115,7 +117,7 @@ public class TestOrcRecordUpdater {
assertEquals(5L, updater.getStats().getRowCount());
Path bucketPath = AcidUtils.createFilename(root, options);
- Path sidePath = OrcRecordUpdater.getSideFile(bucketPath);
+ Path sidePath = OrcAcidUtils.getSideFile(bucketPath);
DataInputStream side = fs.open(sidePath);
// read the stopping point for the first flush and make sure we only see
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcTimezone1.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcTimezone1.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcTimezone1.java
deleted file mode 100644
index 526c357..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcTimezone1.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotNull;
-
-import java.io.File;
-import java.sql.Timestamp;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.TimeZone;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
-import org.apache.hive.common.util.HiveTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import com.google.common.collect.Lists;
-
-/**
- *
- */
-@RunWith(Parameterized.class)
-public class TestOrcTimezone1 {
- Path workDir = new Path(System.getProperty("test.tmp.dir",
- "target" + File.separator + "test" + File.separator + "tmp"));
- Configuration conf;
- FileSystem fs;
- Path testFilePath;
- String writerTimeZone;
- String readerTimeZone;
- static TimeZone defaultTimeZone = TimeZone.getDefault();
-
- public TestOrcTimezone1(String writerTZ, String readerTZ) {
- this.writerTimeZone = writerTZ;
- this.readerTimeZone = readerTZ;
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> data() {
- List<Object[]> result = Arrays.asList(new Object[][]{
- /* Extreme timezones */
- {"GMT-12:00", "GMT+14:00"},
- /* No difference in DST */
- {"America/Los_Angeles", "America/Los_Angeles"}, /* same timezone both with DST */
- {"Europe/Berlin", "Europe/Berlin"}, /* same as above but europe */
- {"America/Phoenix", "Asia/Kolkata"} /* Writer no DST, Reader no DST */,
- {"Europe/Berlin", "America/Los_Angeles"} /* Writer DST, Reader DST */,
- {"Europe/Berlin", "America/Chicago"} /* Writer DST, Reader DST */,
- /* With DST difference */
- {"Europe/Berlin", "UTC"},
- {"UTC", "Europe/Berlin"} /* Writer no DST, Reader DST */,
- {"America/Los_Angeles", "Asia/Kolkata"} /* Writer DST, Reader no DST */,
- {"Europe/Berlin", "Asia/Kolkata"} /* Writer DST, Reader no DST */,
- /* Timezone offsets for the reader has changed historically */
- {"Asia/Saigon", "Pacific/Enderbury"},
- {"UTC", "Asia/Jerusalem"},
-
- // NOTE:
- // "1995-01-01 03:00:00.688888888" this is not a valid time in Pacific/Enderbury timezone.
- // On 1995-01-01 00:00:00 GMT offset moved from -11:00 hr to +13:00 which makes all values
- // on 1995-01-01 invalid. Try this with joda time
- // new MutableDateTime("1995-01-01", DateTimeZone.forTimeZone(readerTimeZone));
- });
- return result;
- }
-
- @Rule
- public TestName testCaseName = new TestName();
-
- @Before
- public void openFileSystem() throws Exception {
- conf = new Configuration();
- fs = FileSystem.getLocal(conf);
- testFilePath = new Path(workDir, "TestOrcFile." +
- testCaseName.getMethodName() + ".orc");
- fs.delete(testFilePath, false);
- }
-
- @After
- public void restoreTimeZone() {
- TimeZone.setDefault(defaultTimeZone);
- }
-
- @Test
- public void testTimestampWriter() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(Timestamp.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- TimeZone.setDefault(TimeZone.getTimeZone(writerTimeZone));
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000));
- assertEquals(writerTimeZone, TimeZone.getDefault().getID());
- List<String> ts = Lists.newArrayList();
- ts.add("2003-01-01 01:00:00.000000222");
- ts.add("1996-08-02 09:00:00.723100809");
- ts.add("1999-01-01 02:00:00.999999999");
- ts.add("1995-01-02 03:00:00.688888888");
- ts.add("2002-01-01 04:00:00.1");
- ts.add("2010-03-02 05:00:00.000009001");
- ts.add("2005-01-01 06:00:00.000002229");
- ts.add("2006-01-01 07:00:00.900203003");
- ts.add("2003-01-01 08:00:00.800000007");
- ts.add("1998-11-02 10:00:00.857340643");
- ts.add("2008-10-02 11:00:00.0");
- ts.add("2037-01-01 00:00:00.000999");
- ts.add("2014-03-28 00:00:00.0");
- for (String t : ts) {
- writer.addRow(Timestamp.valueOf(t));
- }
- writer.close();
-
- TimeZone.setDefault(TimeZone.getTimeZone(readerTimeZone));
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- assertEquals(readerTimeZone, TimeZone.getDefault().getID());
- RecordReader rows = reader.rows(null);
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- Timestamp got = ((TimestampWritable) row).getTimestamp();
- assertEquals(ts.get(idx++), got.toString());
- }
- rows.close();
- }
-
- @Test
- public void testReadTimestampFormat_0_11() throws Exception {
- TimeZone.setDefault(TimeZone.getTimeZone(readerTimeZone));
- Path oldFilePath =
- new Path(HiveTestUtils.getFileFromClasspath("orc-file-11-format.orc"));
- Reader reader = OrcFile.createReader(oldFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
-
- StructObjectInspector readerInspector = (StructObjectInspector) reader
- .getObjectInspector();
- List<? extends StructField> fields = readerInspector
- .getAllStructFieldRefs();
- TimestampObjectInspector tso = (TimestampObjectInspector) readerInspector
- .getStructFieldRef("ts").getFieldObjectInspector();
-
- RecordReader rows = reader.rows();
- Object row = rows.next(null);
- assertNotNull(row);
- assertEquals(Timestamp.valueOf("2000-03-12 15:00:00"),
- tso.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,
- fields.get(12))));
-
- // check the contents of second row
- assertEquals(true, rows.hasNext());
- rows.seekToRow(7499);
- row = rows.next(null);
- assertEquals(Timestamp.valueOf("2000-03-12 15:00:01"),
- tso.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,
- fields.get(12))));
-
- // handle the close up
- assertEquals(false, rows.hasNext());
- rows.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcTimezone2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcTimezone2.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcTimezone2.java
deleted file mode 100644
index 3eae4a9..0000000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcTimezone2.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-
-import java.io.File;
-import java.sql.Timestamp;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Random;
-import java.util.TimeZone;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import com.google.common.collect.Lists;
-
-/**
- *
- */
-@RunWith(Parameterized.class)
-public class TestOrcTimezone2 {
- Path workDir = new Path(System.getProperty("test.tmp.dir",
- "target" + File.separator + "test" + File.separator + "tmp"));
- Configuration conf;
- FileSystem fs;
- Path testFilePath;
- String writerTimeZone;
- String readerTimeZone;
- static TimeZone defaultTimeZone = TimeZone.getDefault();
-
- public TestOrcTimezone2(String writerTZ, String readerTZ) {
- this.writerTimeZone = writerTZ;
- this.readerTimeZone = readerTZ;
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> data() {
- String[] allTimeZones = TimeZone.getAvailableIDs();
- Random rand = new Random(123);
- int len = allTimeZones.length;
- int n = 500;
- Object[][] data = new Object[n][];
- for (int i = 0; i < n; i++) {
- int wIdx = rand.nextInt(len);
- int rIdx = rand.nextInt(len);
- data[i] = new Object[2];
- data[i][0] = allTimeZones[wIdx];
- data[i][1] = allTimeZones[rIdx];
- }
- return Arrays.asList(data);
- }
-
- @Rule
- public TestName testCaseName = new TestName();
-
- @Before
- public void openFileSystem() throws Exception {
- conf = new Configuration();
- fs = FileSystem.getLocal(conf);
- testFilePath = new Path(workDir, "TestOrcFile." +
- testCaseName.getMethodName() + ".orc");
- fs.delete(testFilePath, false);
- }
-
- @After
- public void restoreTimeZone() {
- TimeZone.setDefault(defaultTimeZone);
- }
-
- @Test
- public void testTimestampWriter() throws Exception {
- ObjectInspector inspector;
- synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(Timestamp.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
- }
-
- TimeZone.setDefault(TimeZone.getTimeZone(writerTimeZone));
- Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000));
- assertEquals(writerTimeZone, TimeZone.getDefault().getID());
- List<String> ts = Lists.newArrayList();
- ts.add("2003-01-01 01:00:00.000000222");
- ts.add("1999-01-01 02:00:00.999999999");
- ts.add("1995-01-02 03:00:00.688888888");
- ts.add("2002-01-01 04:00:00.1");
- ts.add("2010-03-02 05:00:00.000009001");
- ts.add("2005-01-01 06:00:00.000002229");
- ts.add("2006-01-01 07:00:00.900203003");
- ts.add("2003-01-01 08:00:00.800000007");
- ts.add("1996-08-02 09:00:00.723100809");
- ts.add("1998-11-02 10:00:00.857340643");
- ts.add("2008-10-02 11:00:00.0");
- ts.add("2037-01-01 00:00:00.000999");
- for (String t : ts) {
- writer.addRow(Timestamp.valueOf(t));
- }
- writer.close();
-
- TimeZone.setDefault(TimeZone.getTimeZone(readerTimeZone));
- Reader reader = OrcFile.createReader(testFilePath,
- OrcFile.readerOptions(conf).filesystem(fs));
- assertEquals(readerTimeZone, TimeZone.getDefault().getID());
- RecordReader rows = reader.rows(null);
- int idx = 0;
- while (rows.hasNext()) {
- Object row = rows.next(null);
- Timestamp got = ((TimestampWritable) row).getTimestamp();
- assertEquals(ts.get(idx++), got.toString());
- }
- rows.close();
- }
-}