You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by do...@apache.org on 2020/11/09 23:11:14 UTC
[orc] branch branch-1.5 updated: ORC-672: FIX ORC type conversion
within arrays where array.length > 1024 (#569)
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/branch-1.5 by this push:
new 1ea9722 ORC-672: FIX ORC type conversion within arrays where array.length > 1024 (#569)
1ea9722 is described below
commit 1ea9722d356aae4ae0ef01bf9974c90b9480d059
Author: Panagiotis Garefalakis <pg...@cloudera.com>
AuthorDate: Mon Nov 9 23:11:05 2020 +0000
ORC-672: FIX ORC type conversion within arrays where array.length > 1024 (#569)
### What changes were proposed in this pull request?
Orc type conversion is throwing ArrayIndexOutOfBoundsException when using ColumnVectors larger than 1024.
The issue is originating from the ConvertTreeReaderFactory where we convert types always using the default CV length instead of the specific batch size.
### Why are the changes needed?
This PR takes into account batchSize when converting types and adds tests for all possible type conversions (with large batchSize) as part of TestConvertTreeReaderFactory.
### How was this patch tested?
TestConvertTreeReaderFactory
---
.../apache/orc/impl/ConvertTreeReaderFactory.java | 42 +--
.../org/apache/orc/TestProlepticConversions.java | 2 +-
.../orc/impl/TestConvertTreeReaderFactory.java | 309 +++++++++++++++++++--
java/core/src/test/resources/bigarray.orc | Bin 2337 -> 0 bytes
4 files changed, 307 insertions(+), 46 deletions(-)
diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index 6bec745..e63de78 100644
--- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -497,7 +497,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (doubleColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- doubleColVector = new DoubleColumnVector();
+ doubleColVector = new DoubleColumnVector(batchSize);
longColVector = (LongColumnVector) previousVector;
}
// Read present/isNull stream
@@ -629,7 +629,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (bytesColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- bytesColVector = new BytesColumnVector();
+ bytesColVector = new BytesColumnVector(batchSize);
longColVector = (LongColumnVector) previousVector;
}
// Read present/isNull stream
@@ -668,7 +668,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (timestampColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- timestampColVector = new TimestampColumnVector();
+ timestampColVector = new TimestampColumnVector(batchSize);
longColVector = (LongColumnVector) previousVector;
}
// Read present/isNull stream
@@ -712,7 +712,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (longColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- longColVector = new LongColumnVector();
+ longColVector = new LongColumnVector(batchSize);
doubleColVector = (DoubleColumnVector) previousVector;
}
// Read present/isNull stream
@@ -793,7 +793,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (bytesColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- bytesColVector = new BytesColumnVector();
+ bytesColVector = new BytesColumnVector(batchSize);
doubleColVector = (DoubleColumnVector) previousVector;
}
// Read present/isNull stream
@@ -833,7 +833,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (timestampColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- timestampColVector = new TimestampColumnVector();
+ timestampColVector = new TimestampColumnVector(batchSize);
doubleColVector = (DoubleColumnVector) previousVector;
}
// Read present/isNull stream
@@ -900,7 +900,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (longColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- longColVector = new LongColumnVector();
+ longColVector = new LongColumnVector(batchSize);
decimalColVector = previousVector;
}
// Read present/isNull stream
@@ -944,7 +944,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (doubleColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- doubleColVector = new DoubleColumnVector();
+ doubleColVector = new DoubleColumnVector(batchSize);
decimalColVector = previousVector;
}
// Read present/isNull stream
@@ -991,7 +991,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (bytesColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- bytesColVector = new BytesColumnVector();
+ bytesColVector = new BytesColumnVector(batchSize);
decimalColVector = previousVector;
}
// Read present/isNull stream
@@ -1042,7 +1042,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (timestampColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- timestampColVector = new TimestampColumnVector();
+ timestampColVector = new TimestampColumnVector(batchSize);
decimalColVector = previousVector;
}
// Read present/isNull stream
@@ -1128,7 +1128,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (longColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- longColVector = new LongColumnVector();
+ longColVector = new LongColumnVector(batchSize);
bytesColVector = (BytesColumnVector) previousVector;
}
// Read present/isNull stream
@@ -1170,7 +1170,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (doubleColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- doubleColVector = new DoubleColumnVector();
+ doubleColVector = new DoubleColumnVector(batchSize);
bytesColVector = (BytesColumnVector) previousVector;
}
// Read present/isNull stream
@@ -1325,7 +1325,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (timestampColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- timestampColVector = new TimestampColumnVector();
+ timestampColVector = new TimestampColumnVector(batchSize);
bytesColVector = (BytesColumnVector) previousVector;
}
// Read present/isNull stream
@@ -1367,7 +1367,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (longColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- longColVector = new DateColumnVector();
+ longColVector = new DateColumnVector(batchSize);
bytesColVector = (BytesColumnVector) previousVector;
}
// Read present/isNull stream
@@ -1463,7 +1463,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (inBytesColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- inBytesColVector = new BytesColumnVector();
+ inBytesColVector = new BytesColumnVector(batchSize);
outBytesColVector = (BytesColumnVector) previousVector;
}
// Read present/isNull stream
@@ -1515,7 +1515,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (longColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- longColVector = new LongColumnVector();
+ longColVector = new LongColumnVector(batchSize);
timestampColVector = (TimestampColumnVector) previousVector;
}
timestampColVector.changeCalendar(fileUsedProlepticGregorian, false);
@@ -1574,7 +1574,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (doubleColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- doubleColVector = new DoubleColumnVector();
+ doubleColVector = new DoubleColumnVector(batchSize);
timestampColVector = (TimestampColumnVector) previousVector;
}
timestampColVector.changeCalendar(fileUsedProlepticGregorian, false);
@@ -1692,7 +1692,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (bytesColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- bytesColVector = new BytesColumnVector();
+ bytesColVector = new BytesColumnVector(batchSize);
timestampColVector = (TimestampColumnVector) previousVector;
}
// Read present/isNull stream
@@ -1739,7 +1739,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (longColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- longColVector = new DateColumnVector();
+ longColVector = new DateColumnVector(batchSize);
timestampColVector = (TimestampColumnVector) previousVector;
}
// Read present/isNull stream
@@ -1786,7 +1786,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (bytesColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- bytesColVector = new BytesColumnVector();
+ bytesColVector = new BytesColumnVector(batchSize);
longColVector = (LongColumnVector) previousVector;
if (longColVector instanceof DateColumnVector) {
dateColumnVector = (DateColumnVector) longColVector;
@@ -1842,7 +1842,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
final int batchSize) throws IOException {
if (timestampColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- timestampColVector = new TimestampColumnVector();
+ timestampColVector = new TimestampColumnVector(batchSize);
longColVector = (LongColumnVector) previousVector;
if (useProlepticGregorian && !(longColVector instanceof DateColumnVector)) {
throw new IllegalArgumentException("Can't use LongColumnVector with" +
diff --git a/java/core/src/test/org/apache/orc/TestProlepticConversions.java b/java/core/src/test/org/apache/orc/TestProlepticConversions.java
index 8320058..f9de685 100644
--- a/java/core/src/test/org/apache/orc/TestProlepticConversions.java
+++ b/java/core/src/test/org/apache/orc/TestProlepticConversions.java
@@ -102,7 +102,7 @@ public class TestProlepticConversions {
fs.delete(testFilePath, false);
}
- private SimpleDateFormat createParser(String format, GregorianCalendar calendar) {
+ public static SimpleDateFormat createParser(String format, GregorianCalendar calendar) {
SimpleDateFormat result = new SimpleDateFormat(format);
result.setCalendar(calendar);
return result;
diff --git a/java/core/src/test/org/apache/orc/impl/TestConvertTreeReaderFactory.java b/java/core/src/test/org/apache/orc/impl/TestConvertTreeReaderFactory.java
index baf5e10..3cb72f6 100644
--- a/java/core/src/test/org/apache/orc/impl/TestConvertTreeReaderFactory.java
+++ b/java/core/src/test/org/apache/orc/impl/TestConvertTreeReaderFactory.java
@@ -17,13 +17,24 @@
*/
package org.apache.orc.impl;
-import static org.junit.Assert.*;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.GregorianCalendar;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DateColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
@@ -32,29 +43,96 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.orc.OrcFile;
import org.apache.orc.Reader;
import org.apache.orc.RecordReader;
-import org.apache.orc.TestVectorOrcFile;
+import org.apache.orc.TestProlepticConversions;
import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class TestConvertTreeReaderFactory {
- @Test
- public void testArraySizeBiggerThan1024AndConvertToDecimal() throws Exception {
- Decimal64ColumnVector columnVector = testArraySizeBiggerThan1024("decimal(6,1)", Decimal64ColumnVector.class);
- assertEquals(columnVector.vector.length, 1025);
+ private Path workDir =
+ new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test" + File.separator + "tmp"));
+
+ private Configuration conf;
+ private FileSystem fs;
+ private Path testFilePath;
+ private int LARGE_BATCH_SIZE;
+
+ @Rule
+ public TestName testCaseName = new TestName();
+
+ @Before
+ public void setupPath() throws Exception {
+ // Default CV length is 1024
+ this.LARGE_BATCH_SIZE = 1030;
+ this.conf = new Configuration();
+ this.fs = FileSystem.getLocal(conf);
+ this.testFilePath = new Path(workDir, TestWriterImpl.class.getSimpleName() + testCaseName.getMethodName().
+ replaceFirst("\\[[0-9]+]", "") + ".orc");
+ fs.delete(testFilePath, false);
}
- public <TExpectedColumn extends ColumnVector> TExpectedColumn testArraySizeBiggerThan1024(
- String typeString, Class<TExpectedColumn> expectedColumnType) throws Exception {
+ public <TExpectedColumnVector extends ColumnVector> TExpectedColumnVector createORCFileWithLargeArray(
+ TypeDescription schema, Class<TExpectedColumnVector> expectedColumnType, boolean useDecimal64)
+ throws IOException, ParseException {
+ conf = new Configuration();
+ fs = FileSystem.getLocal(conf);
+ fs.setWorkingDirectory(workDir);
+ Writer w = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).setSchema(schema));
+
+ SimpleDateFormat dateFormat = TestProlepticConversions.createParser("yyyy-MM-dd", new GregorianCalendar());
+ VectorizedRowBatch batch = schema.createRowBatch(
+ useDecimal64 ? TypeDescription.RowBatchVersion.USE_DECIMAL64 : TypeDescription.RowBatchVersion.ORIGINAL,
+ LARGE_BATCH_SIZE);
+
+ ListColumnVector listCol = (ListColumnVector) batch.cols[0];
+ TExpectedColumnVector dcv = (TExpectedColumnVector) (listCol).child;
+ batch.size = 1;
+ for (int row = 0; row < LARGE_BATCH_SIZE; ++row) {
+ if (dcv instanceof DecimalColumnVector) {
+ ((DecimalColumnVector) dcv).set(row, HiveDecimal.create(row * 2 + 1));
+ } else if (dcv instanceof DoubleColumnVector) {
+ ((DoubleColumnVector) dcv).vector[row] = row * 2 + 1;
+ } else if (dcv instanceof BytesColumnVector) {
+ ((BytesColumnVector) dcv).setVal(row, ((row * 2 + 1) + "").getBytes(StandardCharsets.UTF_8));
+ } else if (dcv instanceof LongColumnVector) {
+ ((LongColumnVector) dcv).vector[row] = row * 2 + 1;
+ } else if (dcv instanceof TimestampColumnVector) {
+ ((TimestampColumnVector) dcv).set(row, Timestamp.valueOf((1900 + row) + "-04-01 12:34:56.9"));
+ } else if (dcv instanceof DateColumnVector) {
+ String date = String.format("%04d-01-23", row * 2 + 1);
+ ((DateColumnVector) dcv).vector[row] = TimeUnit.MILLISECONDS.toDays(dateFormat.parse(date).getTime());
+ } else {
+ throw new IllegalStateException("Writing File with a large array of "+ expectedColumnType + " is not supported!");
+ }
+ }
+
+ listCol.childCount = 1;
+ listCol.lengths[0] = LARGE_BATCH_SIZE;
+ listCol.offsets[0] = 0;
+
+ w.addRowBatch(batch);
+ w.close();
+ assertEquals(((ListColumnVector) batch.cols[0]).child.getClass(), expectedColumnType);
+ return (TExpectedColumnVector) ((ListColumnVector) batch.cols[0]).child;
+ }
+
+ public <TExpectedColumnVector extends ColumnVector> TExpectedColumnVector readORCFileWithLargeArray(
+ String typeString, Class<TExpectedColumnVector> expectedColumnType) throws Exception {
Reader.Options options = new Reader.Options();
- TypeDescription schema = TypeDescription.fromString("struct<col1:array<"+ typeString +">>");
+ TypeDescription schema = TypeDescription.fromString("struct<col1:array<" + typeString + ">>");
options.schema(schema);
String expected = options.toString();
Configuration conf = new Configuration();
- Path path = new Path(TestVectorOrcFile.getFileFromClasspath("bigarray.orc"));
- Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(conf));
+ Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf));
RecordReader rows = reader.rows(options);
VectorizedRowBatch batch = schema.createRowBatchV2();
while (rows.nextBatch(batch)) {
@@ -65,30 +143,213 @@ public class TestConvertTreeReaderFactory {
assertEquals(batch.cols.length, 1);
assertTrue(batch.cols[0] instanceof ListColumnVector);
assertEquals(((ListColumnVector) batch.cols[0]).child.getClass(), expectedColumnType);
- return (TExpectedColumn) ((ListColumnVector) batch.cols[0]).child;
+ return (TExpectedColumnVector) ((ListColumnVector) batch.cols[0]).child;
+ }
+
+ public void testConvertToDecimal() throws Exception {
+ Decimal64ColumnVector columnVector =
+ readORCFileWithLargeArray("decimal(6,1)", Decimal64ColumnVector.class);
+ assertEquals(LARGE_BATCH_SIZE, columnVector.vector.length);
+ }
+
+ public void testConvertToVarchar() throws Exception {
+ BytesColumnVector columnVector = readORCFileWithLargeArray("varchar(10)", BytesColumnVector.class);
+ assertEquals(LARGE_BATCH_SIZE, columnVector.vector.length);
+ }
+
+ public void testConvertToBinary() throws Exception {
+ BytesColumnVector columnVector = readORCFileWithLargeArray("binary", BytesColumnVector.class);
+ assertEquals(LARGE_BATCH_SIZE, columnVector.vector.length);
+ }
+
+ public void testConvertToDouble() throws Exception {
+ DoubleColumnVector columnVector = readORCFileWithLargeArray("double", DoubleColumnVector.class);
+ assertEquals(LARGE_BATCH_SIZE, columnVector.vector.length);
+ }
+
+ public void testConvertToInteger() throws Exception {
+ LongColumnVector columnVector = readORCFileWithLargeArray("int", LongColumnVector.class);
+ assertEquals(LARGE_BATCH_SIZE, columnVector.vector.length);
+ }
+
+ public void testConvertToFloat() throws Exception {
+ DoubleColumnVector columnVector = readORCFileWithLargeArray("float", DoubleColumnVector.class);
+ assertEquals(LARGE_BATCH_SIZE, columnVector.vector.length);
+ }
+
+ public void testConvertToTimestamp() throws Exception {
+ TimestampColumnVector columnVector =
+ readORCFileWithLargeArray("timestamp", TimestampColumnVector.class);
+ assertEquals(LARGE_BATCH_SIZE, columnVector.time.length);
+ }
+
+ public void testConvertToDate() throws Exception {
+ DateColumnVector columnVector = readORCFileWithLargeArray("date", DateColumnVector.class);
+ assertEquals(LARGE_BATCH_SIZE, columnVector.vector.length);
}
@Test
- public void testArraySizeBiggerThan1024AndConvertToVarchar() throws Exception {
- BytesColumnVector columnVector = testArraySizeBiggerThan1024("varchar(10)", BytesColumnVector.class);
- assertEquals(columnVector.vector.length, 1025);
+ public void testDecimalArrayBiggerThanDefault() throws Exception {
+ String typeStr = "decimal(6,1)";
+ Class typeClass = DecimalColumnVector.class;
+
+ TypeDescription schema = TypeDescription.fromString("struct<col1:array<" + typeStr + ">>");
+ createORCFileWithLargeArray(schema, typeClass, typeClass.equals(Decimal64ColumnVector.class));
+ try {
+ // Test all possible conversions
+ // check ConvertTreeReaderFactory.createDecimalConvertTreeReader
+ testConvertToInteger();
+ testConvertToDouble();
+ testConvertToVarchar();
+ testConvertToTimestamp();
+ testConvertToDecimal();
+ } finally {
+ // Make sure we delete file across tests
+ fs.delete(testFilePath, false);
+ }
+ }
+
+ @Test
+ public void testDecimal64ArrayBiggerThanDefault() throws Exception {
+ String typeStr = "decimal(6,1)";
+ Class typeClass = Decimal64ColumnVector.class;
+
+ TypeDescription schema = TypeDescription.fromString("struct<col1:array<" + typeStr + ">>");
+ createORCFileWithLargeArray(schema, typeClass, typeClass.equals(Decimal64ColumnVector.class));
+ try {
+ // Test all possible conversions
+ // check ConvertTreeReaderFactory.createDecimalConvertTreeReader
+ testConvertToInteger();
+ testConvertToDouble();
+ testConvertToVarchar();
+ testConvertToTimestamp();
+ testConvertToDecimal();
+ } finally {
+ // Make sure we delete file across tests
+ fs.delete(testFilePath, false);
+ }
+ }
+
+ @Test
+ public void testStringArrayBiggerThanDefault() throws Exception {
+ String typeStr = "varchar(10)";
+ Class typeClass = BytesColumnVector.class;
+
+ TypeDescription schema = TypeDescription.fromString("struct<col1:array<" + typeStr + ">>");
+ createORCFileWithLargeArray(schema, typeClass, typeClass.equals(Decimal64ColumnVector.class));
+ try {
+ // Test all possible conversions
+ // check ConvertTreeReaderFactory.createStringConvertTreeReader
+ testConvertToInteger();
+ testConvertToDouble();
+ testConvertToDecimal();
+ testConvertToVarchar();
+ testConvertToBinary();
+ testConvertToTimestamp();
+ testConvertToDate();
+ } finally {
+ // Make sure we delete file across tests
+ fs.delete(testFilePath, false);
+ }
+ }
+
+ @Test
+ public void testBinaryArrayBiggerThanDefault() throws Exception {
+ String typeStr = "binary";
+ Class typeClass = BytesColumnVector.class;
+
+ TypeDescription schema = TypeDescription.fromString("struct<col1:array<" + typeStr + ">>");
+ createORCFileWithLargeArray(schema, typeClass, typeClass.equals(Decimal64ColumnVector.class));
+ try {
+ // Test all possible conversions
+ // check ConvertTreeReaderFactory.createBinaryConvertTreeReader
+ testConvertToVarchar();
+ } finally {
+ // Make sure we delete file across tests
+ fs.delete(testFilePath, false);
+ }
+ }
+
+ @Test
+ public void testDoubleArrayBiggerThanDefault() throws Exception {
+ String typeStr = "double";
+ Class typeClass = DoubleColumnVector.class;
+
+ TypeDescription schema = TypeDescription.fromString("struct<col1:array<" + typeStr + ">>");
+ createORCFileWithLargeArray(schema, typeClass, typeClass.equals(Decimal64ColumnVector.class));
+ try {
+ // Test all possible conversions
+ // check ConvertTreeReaderFactory.createDoubleConvertTreeReader
+ testConvertToDouble();
+ testConvertToInteger();
+ testConvertToFloat();
+ testConvertToDecimal();
+ testConvertToVarchar();
+ testConvertToTimestamp();
+ } finally {
+ // Make sure we delete file across tests
+ fs.delete(testFilePath, false);
+ }
}
@Test
- public void testArraySizeBiggerThan1024AndConvertToDouble() throws Exception {
- DoubleColumnVector columnVector = testArraySizeBiggerThan1024("double", DoubleColumnVector.class);
- assertEquals(columnVector.vector.length, 1025);
+ public void testIntArrayBiggerThanDefault() throws Exception {
+ String typeStr = "int";
+ Class typeClass = LongColumnVector.class;
+
+ TypeDescription schema = TypeDescription.fromString("struct<col1:array<" + typeStr + ">>");
+ createORCFileWithLargeArray(schema, typeClass, typeClass.equals(Decimal64ColumnVector.class));
+ try {
+ // Test all possible conversions
+ // check ConvertTreeReaderFactory.createAnyIntegerConvertTreeReader
+ testConvertToInteger();
+ testConvertToDouble();
+ testConvertToDecimal();
+ testConvertToVarchar();
+ testConvertToTimestamp();
+ } finally {
+ // Make sure we delete file across tests
+ fs.delete(testFilePath, false);
+ }
}
@Test
- public void testArraySizeBiggerThan1024AndConvertToInteger() throws Exception {
- LongColumnVector columnVector = testArraySizeBiggerThan1024("int", LongColumnVector.class);
- assertEquals(columnVector.vector.length, 1025);
+ public void testTimestampArrayBiggerThanDefault() throws Exception {
+ String typeStr = "timestamp";
+ Class typeClass = TimestampColumnVector.class;
+
+ TypeDescription schema = TypeDescription.fromString("struct<col1:array<" + typeStr + ">>");
+ createORCFileWithLargeArray(schema, typeClass, typeClass.equals(Decimal64ColumnVector.class));
+ try {
+ // Test all possible conversions
+ // check ConvertTreeReaderFactory.createTimestampConvertTreeReader
+ testConvertToInteger();
+ testConvertToDouble();
+ testConvertToDecimal();
+ testConvertToVarchar();
+ testConvertToTimestamp();
+ testConvertToDate();
+ } finally {
+ // Make sure we delete file across tests
+ fs.delete(testFilePath, false);
+ }
}
@Test
- public void testArraySizeBiggerThan1024AndConvertToTimestamp() throws Exception {
- TimestampColumnVector columnVector = testArraySizeBiggerThan1024("timestamp", TimestampColumnVector.class);
- assertEquals(columnVector.time.length, 1025);
+ public void testDateArrayBiggerThanDefault() throws Exception {
+ String typeStr = "date";
+ Class typeClass = DateColumnVector.class;
+
+ TypeDescription schema = TypeDescription.fromString("struct<col1:array<" + typeStr + ">>");
+ createORCFileWithLargeArray(schema, typeClass, typeClass.equals(Decimal64ColumnVector.class));
+ try {
+ // Test all possible conversions
+ // check ConvertTreeReaderFactory.createDateConvertTreeReader
+ testConvertToVarchar();
+ testConvertToTimestamp();
+ } finally {
+ // Make sure we delete file across tests
+ fs.delete(testFilePath, false);
+ }
}
}
\ No newline at end of file
diff --git a/java/core/src/test/resources/bigarray.orc b/java/core/src/test/resources/bigarray.orc
deleted file mode 100644
index 002565b..0000000
Binary files a/java/core/src/test/resources/bigarray.orc and /dev/null differ