You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2016/08/22 16:07:28 UTC
orc git commit: ORC-96: Pass Context to Orc tree readers (prasanthj
reviewed by omalley)
Repository: orc
Updated Branches:
refs/heads/master 319f94c83 -> d71f13389
ORC-96: Pass Context to Orc tree readers (prasanthj reviewed by omalley)
Fixes #57.
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/d71f1338
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/d71f1338
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/d71f1338
Branch: refs/heads/master
Commit: d71f1338913ef780377ccb4e2e3341a33de4013a
Parents: 319f94c
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Fri Aug 19 18:31:42 2016 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Mon Aug 22 09:06:33 2016 -0700
----------------------------------------------------------------------
.../orc/impl/ConvertTreeReaderFactory.java | 336 ++++++++-----------
.../org/apache/orc/impl/RecordReaderImpl.java | 6 +-
.../org/apache/orc/impl/SchemaEvolution.java | 4 +
.../org/apache/orc/impl/TreeReaderFactory.java | 285 +++++++++-------
4 files changed, 312 insertions(+), 319 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/d71f1338/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index 9290f5c..36b9a20 100644
--- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -52,7 +52,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TreeReader convertTreeReader;
ConvertTreeReader(int columnId) throws IOException {
- super(columnId);
+ super(columnId, null);
}
// The ordering of types here is used to determine which numeric types
@@ -82,10 +82,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
}
protected TreeReader getStringGroupTreeReader(int columnId,
- TypeDescription fileType) throws IOException {
+ TypeDescription fileType, Context context) throws IOException {
switch (fileType.getCategory()) {
case STRING:
- return new StringTreeReader(columnId);
+ return new StringTreeReader(columnId, context);
case CHAR:
return new CharTreeReader(columnId, fileType.getMaxLength());
case VARCHAR:
@@ -403,10 +403,8 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TypeDescription.Category fileTypeCategory;
private TreeReader anyIntegerTreeReader;
- private long longValue;
-
AnyIntegerTreeReader(int columnId, TypeDescription fileType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
this.fileTypeCategory = fileType.getCategory();
switch (fileTypeCategory) {
@@ -417,13 +415,13 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
anyIntegerTreeReader = new ByteTreeReader(columnId);
break;
case SHORT:
- anyIntegerTreeReader = new ShortTreeReader(columnId);
+ anyIntegerTreeReader = new ShortTreeReader(columnId, context);
break;
case INT:
- anyIntegerTreeReader = new IntTreeReader(columnId);
+ anyIntegerTreeReader = new IntTreeReader(columnId, context);
break;
case LONG:
- anyIntegerTreeReader = new LongTreeReader(columnId, skipCorrupt);
+ anyIntegerTreeReader = new LongTreeReader(columnId, context);
break;
default:
throw new RuntimeException("Unexpected type kind " + fileType.getCategory().name());
@@ -431,10 +429,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
setConvertTreeReader(anyIntegerTreeReader);
}
- protected long getLong() throws IOException {
- return longValue;
- }
-
protected String getString(long longValue) {
if (fileTypeCategory == TypeDescription.Category.BOOLEAN) {
return longValue == 0 ? "FALSE" : "TRUE";
@@ -443,10 +437,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
}
}
- protected String getString() {
- return getString(longValue);
- }
-
@Override
public void nextVector(ColumnVector previousVector,
boolean[] isNull,
@@ -462,10 +452,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private final TypeDescription readerType;
private final boolean downCastNeeded;
- AnyIntegerFromAnyIntegerTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, boolean skipCorrupt) throws IOException {
+ AnyIntegerFromAnyIntegerTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType,
+ Context context) throws IOException {
super(columnId);
this.readerType = readerType;
- anyIntegerAsLongTreeReader = new AnyIntegerTreeReader(columnId, fileType, skipCorrupt);
+ anyIntegerAsLongTreeReader = new AnyIntegerTreeReader(columnId, fileType, context);
setConvertTreeReader(anyIntegerAsLongTreeReader);
downCastNeeded = integerDownCastNeeded(fileType, readerType);
}
@@ -599,12 +590,12 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private LongColumnVector longColVector;
AnyIntegerFromDecimalTreeReader(int columnId, TypeDescription fileType,
- TypeDescription readerType) throws IOException {
+ TypeDescription readerType, Context context) throws IOException {
super(columnId);
this.precision = fileType.getPrecision();
this.scale = fileType.getScale();
this.readerType = readerType;
- decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
+ decimalTreeReader = new DecimalTreeReader(columnId, context);
setConvertTreeReader(decimalTreeReader);
}
@@ -648,10 +639,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private LongColumnVector longColVector;
AnyIntegerFromStringGroupTreeReader(int columnId, TypeDescription fileType,
- TypeDescription readerType) throws IOException {
+ TypeDescription readerType, Context context) throws IOException {
super(columnId);
this.readerType = readerType;
- stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
+ stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType, context);
setConvertTreeReader(stringGroupTreeReader);
}
@@ -692,10 +683,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private LongColumnVector longColVector;
AnyIntegerFromTimestampTreeReader(int columnId, TypeDescription readerType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
this.readerType = readerType;
- timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
+ timestampTreeReader = new TimestampTreeReader(columnId, context);
setConvertTreeReader(timestampTreeReader);
}
@@ -731,10 +722,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private DoubleColumnVector doubleColVector;
FloatFromAnyIntegerTreeReader(int columnId, TypeDescription fileType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
anyIntegerAsLongTreeReader =
- new AnyIntegerTreeReader(columnId, fileType, skipCorrupt);
+ new AnyIntegerTreeReader(columnId, fileType, context);
setConvertTreeReader(anyIntegerAsLongTreeReader);
}
@@ -816,11 +807,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private DoubleColumnVector doubleColVector;
FloatFromDecimalTreeReader(int columnId, TypeDescription fileType,
- TypeDescription readerType) throws IOException {
+ TypeDescription readerType, Context context) throws IOException {
super(columnId);
this.precision = fileType.getPrecision();
this.scale = fileType.getScale();
- decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
+ decimalTreeReader = new DecimalTreeReader(columnId, context);
setConvertTreeReader(decimalTreeReader);
}
@@ -853,10 +844,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private BytesColumnVector bytesColVector;
private DoubleColumnVector doubleColVector;
- FloatFromStringGroupTreeReader(int columnId, TypeDescription fileType)
+ FloatFromStringGroupTreeReader(int columnId, TypeDescription fileType, Context context)
throws IOException {
super(columnId);
- stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
+ stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType, context);
setConvertTreeReader(stringGroupTreeReader);
}
@@ -896,9 +887,9 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TimestampColumnVector timestampColVector;
private DoubleColumnVector doubleColVector;
- FloatFromTimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException {
+ FloatFromTimestampTreeReader(int columnId, Context context) throws IOException {
super(columnId);
- timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
+ timestampTreeReader = new TimestampTreeReader(columnId, context);
setConvertTreeReader(timestampTreeReader);
}
@@ -932,10 +923,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private DoubleColumnVector doubleColVector;
DoubleFromAnyIntegerTreeReader(int columnId, TypeDescription fileType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
anyIntegerAsLongTreeReader =
- new AnyIntegerTreeReader(columnId, fileType, skipCorrupt);
+ new AnyIntegerTreeReader(columnId, fileType, context);
setConvertTreeReader(anyIntegerAsLongTreeReader);
}
@@ -1022,11 +1013,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private DecimalColumnVector decimalColVector;
private DoubleColumnVector doubleColVector;
- DoubleFromDecimalTreeReader(int columnId, TypeDescription fileType) throws IOException {
+ DoubleFromDecimalTreeReader(int columnId, TypeDescription fileType, Context context) throws IOException {
super(columnId);
this.precision = fileType.getPrecision();
this.scale = fileType.getScale();
- decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
+ decimalTreeReader = new DecimalTreeReader(columnId, context);
setConvertTreeReader(decimalTreeReader);
}
@@ -1059,10 +1050,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private BytesColumnVector bytesColVector;
private DoubleColumnVector doubleColVector;
- DoubleFromStringGroupTreeReader(int columnId, TypeDescription fileType)
+ DoubleFromStringGroupTreeReader(int columnId, TypeDescription fileType, Context context)
throws IOException {
super(columnId);
- stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
+ stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType, context);
setConvertTreeReader(stringGroupTreeReader);
}
@@ -1101,9 +1092,9 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TimestampColumnVector timestampColVector;
private DoubleColumnVector doubleColVector;
- DoubleFromTimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException {
+ DoubleFromTimestampTreeReader(int columnId, Context context) throws IOException {
super(columnId);
- timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
+ timestampTreeReader = new TimestampTreeReader(columnId, context);
setConvertTreeReader(timestampTreeReader);
}
@@ -1136,11 +1127,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private LongColumnVector longColVector;
private DecimalColumnVector decimalColVector;
- DecimalFromAnyIntegerTreeReader(int columnId, TypeDescription fileType, boolean skipCorrupt)
+ DecimalFromAnyIntegerTreeReader(int columnId, TypeDescription fileType, Context context)
throws IOException {
super(columnId);
anyIntegerAsLongTreeReader =
- new AnyIntegerTreeReader(columnId, fileType, skipCorrupt);
+ new AnyIntegerTreeReader(columnId, fileType, context);
setConvertTreeReader(anyIntegerAsLongTreeReader);
}
@@ -1267,9 +1258,9 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private DecimalColumnVector decimalColVector;
DecimalFromStringGroupTreeReader(int columnId, TypeDescription fileType,
- TypeDescription readerType) throws IOException {
+ TypeDescription readerType, Context context) throws IOException {
super(columnId);
- stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
+ stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType, context);
setConvertTreeReader(stringGroupTreeReader);
}
@@ -1309,9 +1300,9 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TimestampColumnVector timestampColVector;
private DecimalColumnVector decimalColVector;
- DecimalFromTimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException {
+ DecimalFromTimestampTreeReader(int columnId, Context context) throws IOException {
super(columnId);
- timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
+ timestampTreeReader = new TimestampTreeReader(columnId, context);
setConvertTreeReader(timestampTreeReader);
}
@@ -1356,14 +1347,14 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private int readerScale;
private DecimalColumnVector decimalColVector;
- DecimalFromDecimalTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType)
+ DecimalFromDecimalTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, Context context)
throws IOException {
super(columnId);
filePrecision = fileType.getPrecision();
fileScale = fileType.getScale();
readerPrecision = readerType.getPrecision();
readerScale = readerType.getScale();
- decimalTreeReader = new DecimalTreeReader(columnId, filePrecision, fileScale);
+ decimalTreeReader = new DecimalTreeReader(columnId, context);
setConvertTreeReader(decimalTreeReader);
}
@@ -1405,11 +1396,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private BytesColumnVector bytesColVector;
StringGroupFromAnyIntegerTreeReader(int columnId, TypeDescription fileType,
- TypeDescription readerType, boolean skipCorrupt) throws IOException {
+ TypeDescription readerType, Context context) throws IOException {
super(columnId);
this.readerType = readerType;
anyIntegerAsLongTreeReader =
- new AnyIntegerTreeReader(columnId, fileType, skipCorrupt);
+ new AnyIntegerTreeReader(columnId, fileType, context);
setConvertTreeReader(anyIntegerAsLongTreeReader);
}
@@ -1447,7 +1438,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
StringGroupFromFloatTreeReader(int columnId, TypeDescription readerType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
this.readerType = readerType;
floatTreeReader = new FloatTreeReader(columnId);
@@ -1492,7 +1483,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private BytesColumnVector bytesColVector;
StringGroupFromDoubleTreeReader(int columnId, TypeDescription readerType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
this.readerType = readerType;
doubleTreeReader = new DoubleTreeReader(columnId);
@@ -1541,12 +1532,12 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private BytesColumnVector bytesColVector;
StringGroupFromDecimalTreeReader(int columnId, TypeDescription fileType,
- TypeDescription readerType, boolean skipCorrupt) throws IOException {
+ TypeDescription readerType, Context context) throws IOException {
super(columnId);
this.precision = fileType.getPrecision();
this.scale = fileType.getScale();
this.readerType = readerType;
- decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
+ decimalTreeReader = new DecimalTreeReader(columnId, context);
setConvertTreeReader(decimalTreeReader);
}
@@ -1582,10 +1573,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private BytesColumnVector bytesColVector;
StringGroupFromTimestampTreeReader(int columnId, TypeDescription readerType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
this.readerType = readerType;
- timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
+ timestampTreeReader = new TimestampTreeReader(columnId, context);
setConvertTreeReader(timestampTreeReader);
}
@@ -1623,10 +1614,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private Date date;
StringGroupFromDateTreeReader(int columnId, TypeDescription readerType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
this.readerType = readerType;
- dateTreeReader = new DateTreeReader(columnId);
+ dateTreeReader = new DateTreeReader(columnId, context);
setConvertTreeReader(dateTreeReader);
date = new Date(0);
}
@@ -1662,10 +1653,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private final TypeDescription readerType;
StringGroupFromStringGroupTreeReader(int columnId, TypeDescription fileType,
- TypeDescription readerType) throws IOException {
+ TypeDescription readerType, Context context) throws IOException {
super(columnId);
this.readerType = readerType;
- stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
+ stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType, context);
setConvertTreeReader(stringGroupTreeReader);
}
@@ -1708,10 +1699,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private BytesColumnVector outBytesColVector;
StringGroupFromBinaryTreeReader(int columnId, TypeDescription readerType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
this.readerType = readerType;
- binaryTreeReader = new BinaryTreeReader(columnId);
+ binaryTreeReader = new BinaryTreeReader(columnId, context);
setConvertTreeReader(binaryTreeReader);
}
@@ -1759,10 +1750,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TimestampColumnVector timestampColVector;
TimestampFromAnyIntegerTreeReader(int columnId, TypeDescription fileType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
anyIntegerAsLongTreeReader =
- new AnyIntegerTreeReader(columnId, fileType, skipCorrupt);
+ new AnyIntegerTreeReader(columnId, fileType, context);
setConvertTreeReader(anyIntegerAsLongTreeReader);
}
@@ -1797,7 +1788,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TimestampColumnVector timestampColVector;
TimestampFromFloatTreeReader(int columnId, TypeDescription fileType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
floatTreeReader = new FloatTreeReader(columnId);
setConvertTreeReader(floatTreeReader);
@@ -1835,7 +1826,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TimestampColumnVector timestampColVector;
TimestampFromDoubleTreeReader(int columnId, TypeDescription fileType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
doubleTreeReader = new DoubleTreeReader(columnId);
setConvertTreeReader(doubleTreeReader);
@@ -1875,11 +1866,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TimestampColumnVector timestampColVector;
TimestampFromDecimalTreeReader(int columnId, TypeDescription fileType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
this.precision = fileType.getPrecision();
this.scale = fileType.getScale();
- decimalTreeReader = new DecimalTreeReader(columnId, precision, scale);
+ decimalTreeReader = new DecimalTreeReader(columnId, context);
setConvertTreeReader(decimalTreeReader);
}
@@ -1915,10 +1906,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private BytesColumnVector bytesColVector;
private TimestampColumnVector timestampColVector;
- TimestampFromStringGroupTreeReader(int columnId, TypeDescription fileType)
+ TimestampFromStringGroupTreeReader(int columnId, TypeDescription fileType, Context context)
throws IOException {
super(columnId);
- stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
+ stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType, context);
setConvertTreeReader(stringGroupTreeReader);
}
@@ -1959,9 +1950,9 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TimestampColumnVector timestampColVector;
TimestampFromDateTreeReader(int columnId, TypeDescription fileType,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
super(columnId);
- dateTreeReader = new DateTreeReader(columnId);
+ dateTreeReader = new DateTreeReader(columnId, context);
setConvertTreeReader(dateTreeReader);
}
@@ -1995,10 +1986,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private BytesColumnVector bytesColVector;
private LongColumnVector longColVector;
- DateFromStringGroupTreeReader(int columnId, TypeDescription fileType)
+ DateFromStringGroupTreeReader(int columnId, TypeDescription fileType, Context context)
throws IOException {
super(columnId);
- stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
+ stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType, context);
setConvertTreeReader(stringGroupTreeReader);
}
@@ -2038,9 +2029,9 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TimestampColumnVector timestampColVector;
private LongColumnVector longColVector;
- DateFromTimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException {
+ DateFromTimestampTreeReader(int columnId, Context context) throws IOException {
super(columnId);
- timestampTreeReader = new TimestampTreeReader(columnId, skipCorrupt);
+ timestampTreeReader = new TimestampTreeReader(columnId, context);
setConvertTreeReader(timestampTreeReader);
}
@@ -2072,10 +2063,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private TreeReader stringGroupTreeReader;
- BinaryFromStringGroupTreeReader(int columnId, TypeDescription fileType)
+ BinaryFromStringGroupTreeReader(int columnId, TypeDescription fileType, Context context)
throws IOException {
super(columnId);
- stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType);
+ stringGroupTreeReader = getStringGroupTreeReader(columnId, fileType, context);
setConvertTreeReader(stringGroupTreeReader);
}
@@ -2090,9 +2081,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createAnyIntegerConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from (BOOLEAN, BYTE, SHORT, INT, LONG) to schema type.
//
@@ -2108,27 +2097,27 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
readerType.getCategory() + " to self needed");
}
return new AnyIntegerFromAnyIntegerTreeReader(columnId, fileType, readerType,
- skipCorrupt);
+ context);
case FLOAT:
return new FloatFromAnyIntegerTreeReader(columnId, fileType,
- skipCorrupt);
+ context);
case DOUBLE:
return new DoubleFromAnyIntegerTreeReader(columnId, fileType,
- skipCorrupt);
+ context);
case DECIMAL:
- return new DecimalFromAnyIntegerTreeReader(columnId, fileType, skipCorrupt);
+ return new DecimalFromAnyIntegerTreeReader(columnId, fileType, context);
case STRING:
case CHAR:
case VARCHAR:
return new StringGroupFromAnyIntegerTreeReader(columnId, fileType, readerType,
- skipCorrupt);
+ context);
case TIMESTAMP:
- return new TimestampFromAnyIntegerTreeReader(columnId, fileType, skipCorrupt);
+ return new TimestampFromAnyIntegerTreeReader(columnId, fileType, context);
// Not currently supported conversion(s):
case BINARY:
@@ -2147,9 +2136,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createFloatConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from FLOAT to schema type.
switch (readerType.getCategory()) {
@@ -2174,10 +2161,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case STRING:
case CHAR:
case VARCHAR:
- return new StringGroupFromFloatTreeReader(columnId, readerType, skipCorrupt);
+ return new StringGroupFromFloatTreeReader(columnId, readerType, context);
case TIMESTAMP:
- return new TimestampFromFloatTreeReader(columnId, readerType, skipCorrupt);
+ return new TimestampFromFloatTreeReader(columnId, readerType, context);
// Not currently supported conversion(s):
case BINARY:
@@ -2196,9 +2183,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createDoubleConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from DOUBLE to schema type.
switch (readerType.getCategory()) {
@@ -2223,10 +2208,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case STRING:
case CHAR:
case VARCHAR:
- return new StringGroupFromDoubleTreeReader(columnId, readerType, skipCorrupt);
+ return new StringGroupFromDoubleTreeReader(columnId, readerType, context);
case TIMESTAMP:
- return new TimestampFromDoubleTreeReader(columnId, readerType, skipCorrupt);
+ return new TimestampFromDoubleTreeReader(columnId, readerType, context);
// Not currently supported conversion(s):
case BINARY:
@@ -2245,9 +2230,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createDecimalConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from DECIMAL to schema type.
switch (readerType.getCategory()) {
@@ -2257,24 +2240,24 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case SHORT:
case INT:
case LONG:
- return new AnyIntegerFromDecimalTreeReader(columnId, fileType, readerType);
+ return new AnyIntegerFromDecimalTreeReader(columnId, fileType, readerType, context);
case FLOAT:
- return new FloatFromDecimalTreeReader(columnId, fileType, readerType);
+ return new FloatFromDecimalTreeReader(columnId, fileType, readerType, context);
case DOUBLE:
- return new DoubleFromDecimalTreeReader(columnId, fileType);
+ return new DoubleFromDecimalTreeReader(columnId, fileType, context);
case STRING:
case CHAR:
case VARCHAR:
- return new StringGroupFromDecimalTreeReader(columnId, fileType, readerType, skipCorrupt);
+ return new StringGroupFromDecimalTreeReader(columnId, fileType, readerType, context);
case TIMESTAMP:
- return new TimestampFromDecimalTreeReader(columnId, fileType, skipCorrupt);
+ return new TimestampFromDecimalTreeReader(columnId, fileType, context);
case DECIMAL:
- return new DecimalFromDecimalTreeReader(columnId, fileType, readerType);
+ return new DecimalFromDecimalTreeReader(columnId, fileType, readerType, context);
// Not currently supported conversion(s):
case BINARY:
@@ -2293,9 +2276,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createStringConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from STRING to schema type.
switch (readerType.getCategory()) {
@@ -2305,35 +2286,35 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case SHORT:
case INT:
case LONG:
- return new AnyIntegerFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new AnyIntegerFromStringGroupTreeReader(columnId, fileType, readerType, context);
case FLOAT:
- return new FloatFromStringGroupTreeReader(columnId, fileType);
+ return new FloatFromStringGroupTreeReader(columnId, fileType, context);
case DOUBLE:
- return new DoubleFromStringGroupTreeReader(columnId, fileType);
+ return new DoubleFromStringGroupTreeReader(columnId, fileType, context);
case DECIMAL:
- return new DecimalFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new DecimalFromStringGroupTreeReader(columnId, fileType, readerType, context);
case CHAR:
- return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType, context);
case VARCHAR:
- return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType, context);
case STRING:
throw new IllegalArgumentException("No conversion of type " +
readerType.getCategory() + " to self needed");
case BINARY:
- return new BinaryFromStringGroupTreeReader(columnId, fileType);
+ return new BinaryFromStringGroupTreeReader(columnId, fileType, context);
case TIMESTAMP:
- return new TimestampFromStringGroupTreeReader(columnId, fileType);
+ return new TimestampFromStringGroupTreeReader(columnId, fileType, context);
case DATE:
- return new DateFromStringGroupTreeReader(columnId, fileType);
+ return new DateFromStringGroupTreeReader(columnId, fileType, context);
// Not currently supported conversion(s):
@@ -2350,9 +2331,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createCharConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from CHAR to schema type.
switch (readerType.getCategory()) {
@@ -2362,34 +2341,34 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case SHORT:
case INT:
case LONG:
- return new AnyIntegerFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new AnyIntegerFromStringGroupTreeReader(columnId, fileType, readerType, context);
case FLOAT:
- return new FloatFromStringGroupTreeReader(columnId, fileType);
+ return new FloatFromStringGroupTreeReader(columnId, fileType, context);
case DOUBLE:
- return new DoubleFromStringGroupTreeReader(columnId, fileType);
+ return new DoubleFromStringGroupTreeReader(columnId, fileType, context);
case DECIMAL:
- return new DecimalFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new DecimalFromStringGroupTreeReader(columnId, fileType, readerType, context);
case STRING:
- return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType, context);
case VARCHAR:
- return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType, context);
case CHAR:
- return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType, context);
case BINARY:
- return new BinaryFromStringGroupTreeReader(columnId, fileType);
+ return new BinaryFromStringGroupTreeReader(columnId, fileType, context);
case TIMESTAMP:
- return new TimestampFromStringGroupTreeReader(columnId, fileType);
+ return new TimestampFromStringGroupTreeReader(columnId, fileType, context);
case DATE:
- return new DateFromStringGroupTreeReader(columnId, fileType);
+ return new DateFromStringGroupTreeReader(columnId, fileType, context);
// Not currently supported conversion(s):
@@ -2406,9 +2385,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createVarcharConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from VARCHAR to schema type.
switch (readerType.getCategory()) {
@@ -2418,34 +2395,34 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case SHORT:
case INT:
case LONG:
- return new AnyIntegerFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new AnyIntegerFromStringGroupTreeReader(columnId, fileType, readerType, context);
case FLOAT:
- return new FloatFromStringGroupTreeReader(columnId, fileType);
+ return new FloatFromStringGroupTreeReader(columnId, fileType, context);
case DOUBLE:
- return new DoubleFromStringGroupTreeReader(columnId, fileType);
+ return new DoubleFromStringGroupTreeReader(columnId, fileType, context);
case DECIMAL:
- return new DecimalFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new DecimalFromStringGroupTreeReader(columnId, fileType, readerType, context);
case STRING:
- return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType, context);
case CHAR:
- return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType, context);
case VARCHAR:
- return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType, context);
case BINARY:
- return new BinaryFromStringGroupTreeReader(columnId, fileType);
+ return new BinaryFromStringGroupTreeReader(columnId, fileType, context);
case TIMESTAMP:
- return new TimestampFromStringGroupTreeReader(columnId, fileType);
+ return new TimestampFromStringGroupTreeReader(columnId, fileType, context);
case DATE:
- return new DateFromStringGroupTreeReader(columnId, fileType);
+ return new DateFromStringGroupTreeReader(columnId, fileType, context);
// Not currently supported conversion(s):
@@ -2462,9 +2439,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createTimestampConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from TIMESTAMP to schema type.
switch (readerType.getCategory()) {
@@ -2474,28 +2449,28 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case SHORT:
case INT:
case LONG:
- return new AnyIntegerFromTimestampTreeReader(columnId, readerType, skipCorrupt);
+ return new AnyIntegerFromTimestampTreeReader(columnId, readerType, context);
case FLOAT:
- return new FloatFromTimestampTreeReader(columnId, skipCorrupt);
+ return new FloatFromTimestampTreeReader(columnId, context);
case DOUBLE:
- return new DoubleFromTimestampTreeReader(columnId, skipCorrupt);
+ return new DoubleFromTimestampTreeReader(columnId, context);
case DECIMAL:
- return new DecimalFromTimestampTreeReader(columnId, skipCorrupt);
+ return new DecimalFromTimestampTreeReader(columnId, context);
case STRING:
case CHAR:
case VARCHAR:
- return new StringGroupFromTimestampTreeReader(columnId, readerType, skipCorrupt);
+ return new StringGroupFromTimestampTreeReader(columnId, readerType, context);
case TIMESTAMP:
throw new IllegalArgumentException("No conversion of type " +
readerType.getCategory() + " to self needed");
case DATE:
- return new DateFromTimestampTreeReader(columnId, skipCorrupt);
+ return new DateFromTimestampTreeReader(columnId, context);
// Not currently supported conversion(s):
case BINARY:
@@ -2513,9 +2488,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createDateConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from DATE to schema type.
switch (readerType.getCategory()) {
@@ -2523,10 +2496,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case STRING:
case CHAR:
case VARCHAR:
- return new StringGroupFromDateTreeReader(columnId, readerType, skipCorrupt);
+ return new StringGroupFromDateTreeReader(columnId, readerType, context);
case TIMESTAMP:
- return new TimestampFromDateTreeReader(columnId, readerType, skipCorrupt);
+ return new TimestampFromDateTreeReader(columnId, readerType, context);
case DATE:
throw new IllegalArgumentException("No conversion of type " +
@@ -2556,9 +2529,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private static TreeReader createBinaryConvertTreeReader(int columnId,
TypeDescription fileType,
TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
+ Context context) throws IOException {
// CONVERT from DATE to schema type.
switch (readerType.getCategory()) {
@@ -2566,7 +2537,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case STRING:
case CHAR:
case VARCHAR:
- return new StringGroupFromBinaryTreeReader(columnId, readerType, skipCorrupt);
+ return new StringGroupFromBinaryTreeReader(columnId, readerType, context);
case BINARY:
throw new IllegalArgumentException("No conversion of type " +
@@ -2715,17 +2686,12 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
* Convert value for tag
*
* @param readerType
- * @param evolution
- * @param included
- * @param skipCorrupt
* @return
* @throws IOException
*/
public static TreeReader createConvertTreeReader(TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt
- ) throws IOException {
+ Context context) throws IOException {
+ final SchemaEvolution evolution = context.getSchemaEvolution();
int columnId = readerType.getId();
TypeDescription fileType = evolution.getFileType(readerType);
@@ -2737,44 +2703,34 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
case SHORT:
case INT:
case LONG:
- return createAnyIntegerConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createAnyIntegerConvertTreeReader(columnId, fileType, readerType, context);
case FLOAT:
- return createFloatConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createFloatConvertTreeReader(columnId, fileType, readerType, context);
case DOUBLE:
- return createDoubleConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createDoubleConvertTreeReader(columnId, fileType, readerType, context);
case DECIMAL:
- return createDecimalConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createDecimalConvertTreeReader(columnId, fileType, readerType, context);
case STRING:
- return createStringConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createStringConvertTreeReader(columnId, fileType, readerType, context);
case CHAR:
- return createCharConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createCharConvertTreeReader(columnId, fileType, readerType, context);
case VARCHAR:
- return createVarcharConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createVarcharConvertTreeReader(columnId, fileType, readerType, context);
case TIMESTAMP:
- return createTimestampConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createTimestampConvertTreeReader(columnId, fileType, readerType, context);
case DATE:
- return createDateConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createDateConvertTreeReader(columnId, fileType, readerType, context);
case BINARY:
- return createBinaryConvertTreeReader(columnId, fileType, readerType, evolution,
- included, skipCorrupt);
+ return createBinaryConvertTreeReader(columnId, fileType, readerType, context);
// UNDONE: Complex conversions...
case STRUCT:
http://git-wip-us.apache.org/repos/asf/orc/blob/d71f1338/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
index 8a492dc..e8ad54d 100644
--- a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
@@ -210,8 +210,10 @@ public class RecordReaderImpl implements RecordReader {
skipCorrupt = OrcConf.SKIP_CORRUPT_DATA.getBoolean(fileReader.conf);
}
- reader = TreeReaderFactory.createTreeReader(evolution.getReaderSchema(),
- evolution, included, skipCorrupt);
+ TreeReaderFactory.ReaderContext readerContext = new TreeReaderFactory.ReaderContext()
+ .setSchemaEvolution(evolution)
+ .skipCorrupt(skipCorrupt);
+ reader = TreeReaderFactory.createTreeReader(evolution.getReaderSchema(), readerContext);
writerIncluded = evolution.getFileIncluded();
indexes = new OrcProto.RowIndex[types.size()];
http://git-wip-us.apache.org/repos/asf/orc/blob/d71f1338/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
index 364df9e..1e11728 100644
--- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
+++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
@@ -166,6 +166,10 @@ public class SchemaEvolution {
return readerFileTypes[id];
}
+ public boolean[] getReaderIncluded() {
+ return readerIncluded;
+ }
+
public boolean[] getFileIncluded() {
return fileIncluded;
}
http://git-wip-us.apache.org/repos/asf/orc/blob/d71f1338/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
index bbba932..0c94f0e 100644
--- a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
@@ -48,18 +48,64 @@ import org.apache.orc.OrcProto;
*/
public class TreeReaderFactory {
+ public interface Context {
+ SchemaEvolution getSchemaEvolution();
+
+ boolean isSkipCorrupt();
+
+ String getWriterTimezone();
+ }
+
+ public static class ReaderContext implements Context {
+ private SchemaEvolution evolution;
+ private boolean skipCorrupt = false;
+ private String writerTimezone;
+
+ public ReaderContext setSchemaEvolution(SchemaEvolution evolution) {
+ this.evolution = evolution;
+ return this;
+ }
+
+ public ReaderContext skipCorrupt(boolean skipCorrupt) {
+ this.skipCorrupt = skipCorrupt;
+ return this;
+ }
+
+ public ReaderContext writerTimeZone(String writerTimezone) {
+ this.writerTimezone = writerTimezone;
+ return this;
+ }
+
+ @Override
+ public SchemaEvolution getSchemaEvolution() {
+ return evolution;
+ }
+
+ @Override
+ public boolean isSkipCorrupt() {
+ return skipCorrupt;
+ }
+
+ @Override
+ public String getWriterTimezone() {
+ return writerTimezone;
+ }
+ }
+
public abstract static class TreeReader {
protected final int columnId;
protected BitFieldReader present = null;
protected boolean valuePresent = false;
protected int vectorColumnCount;
+ protected final Context context;
- TreeReader(int columnId) throws IOException {
- this(columnId, null);
+ TreeReader(int columnId, Context context) throws IOException {
+ this(columnId, null, context);
}
- protected TreeReader(int columnId, InStream in) throws IOException {
+ protected TreeReader(int columnId, InStream in, Context context) throws IOException {
this.columnId = columnId;
+ this.context = context;
if (in == null) {
present = null;
valuePresent = true;
@@ -82,11 +128,12 @@ public class TreeReaderFactory {
static IntegerReader createIntegerReader(OrcProto.ColumnEncoding.Kind kind,
InStream in,
- boolean signed, boolean skipCorrupt) throws IOException {
+ boolean signed,
+ Context context) throws IOException {
switch (kind) {
case DIRECT_V2:
case DICTIONARY_V2:
- return new RunLengthIntegerReaderV2(in, signed, skipCorrupt);
+ return new RunLengthIntegerReaderV2(in, signed, context == null ? false : context.isSkipCorrupt());
case DIRECT:
case DICTIONARY:
return new RunLengthIntegerReader(in, signed);
@@ -206,7 +253,7 @@ public class TreeReaderFactory {
public static class NullTreeReader extends TreeReader {
public NullTreeReader(int columnId) throws IOException {
- super(columnId);
+ super(columnId, null);
}
@Override
@@ -246,7 +293,7 @@ public class TreeReaderFactory {
}
protected BooleanTreeReader(int columnId, InStream present, InStream data) throws IOException {
- super(columnId, present);
+ super(columnId, present, null);
if (data != null) {
reader = new BitFieldReader(data, 1);
}
@@ -299,7 +346,7 @@ public class TreeReaderFactory {
}
protected ByteTreeReader(int columnId, InStream present, InStream data) throws IOException {
- super(columnId, present);
+ super(columnId, present, null);
this.reader = new RunLengthByteReader(data);
}
@@ -345,17 +392,17 @@ public class TreeReaderFactory {
public static class ShortTreeReader extends TreeReader {
protected IntegerReader reader = null;
- ShortTreeReader(int columnId) throws IOException {
- this(columnId, null, null, null);
+ ShortTreeReader(int columnId, Context context) throws IOException {
+ this(columnId, null, null, null, context);
}
protected ShortTreeReader(int columnId, InStream present, InStream data,
- OrcProto.ColumnEncoding encoding)
+ OrcProto.ColumnEncoding encoding, Context context)
throws IOException {
- super(columnId, present);
+ super(columnId, present, context);
if (data != null && encoding != null) {
checkEncoding(encoding);
- this.reader = createIntegerReader(encoding.getKind(), data, true, false);
+ this.reader = createIntegerReader(encoding.getKind(), data, true, context);
}
}
@@ -376,7 +423,7 @@ public class TreeReaderFactory {
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
- streams.get(name), true, false);
+ streams.get(name), true, context);
}
@Override
@@ -412,17 +459,17 @@ public class TreeReaderFactory {
public static class IntTreeReader extends TreeReader {
protected IntegerReader reader = null;
- IntTreeReader(int columnId) throws IOException {
- this(columnId, null, null, null);
+ IntTreeReader(int columnId, Context context) throws IOException {
+ this(columnId, null, null, null, context);
}
protected IntTreeReader(int columnId, InStream present, InStream data,
- OrcProto.ColumnEncoding encoding)
+ OrcProto.ColumnEncoding encoding, Context context)
throws IOException {
- super(columnId, present);
+ super(columnId, present, context);
if (data != null && encoding != null) {
checkEncoding(encoding);
- this.reader = createIntegerReader(encoding.getKind(), data, true, false);
+ this.reader = createIntegerReader(encoding.getKind(), data, true, context);
}
}
@@ -443,7 +490,7 @@ public class TreeReaderFactory {
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
- streams.get(name), true, false);
+ streams.get(name), true, context);
}
@Override
@@ -479,18 +526,18 @@ public class TreeReaderFactory {
public static class LongTreeReader extends TreeReader {
protected IntegerReader reader = null;
- LongTreeReader(int columnId, boolean skipCorrupt) throws IOException {
- this(columnId, null, null, null, skipCorrupt);
+ LongTreeReader(int columnId, Context context) throws IOException {
+ this(columnId, null, null, null, context);
}
protected LongTreeReader(int columnId, InStream present, InStream data,
OrcProto.ColumnEncoding encoding,
- boolean skipCorrupt)
+ Context context)
throws IOException {
- super(columnId, present);
+ super(columnId, present, context);
if (data != null && encoding != null) {
checkEncoding(encoding);
- this.reader = createIntegerReader(encoding.getKind(), data, true, skipCorrupt);
+ this.reader = createIntegerReader(encoding.getKind(), data, true, context);
}
}
@@ -511,7 +558,7 @@ public class TreeReaderFactory {
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
- streams.get(name), true, false);
+ streams.get(name), true, context);
}
@Override
@@ -553,7 +600,7 @@ public class TreeReaderFactory {
}
protected FloatTreeReader(int columnId, InStream present, InStream data) throws IOException {
- super(columnId, present);
+ super(columnId, present, null);
this.utils = new SerializationUtils();
this.stream = data;
}
@@ -646,7 +693,7 @@ public class TreeReaderFactory {
}
protected DoubleTreeReader(int columnId, InStream present, InStream data) throws IOException {
- super(columnId, present);
+ super(columnId, present, null);
this.utils = new SerializationUtils();
this.stream = data;
}
@@ -737,18 +784,18 @@ public class TreeReaderFactory {
protected IntegerReader lengths = null;
protected final LongColumnVector scratchlcv;
- BinaryTreeReader(int columnId) throws IOException {
- this(columnId, null, null, null, null);
+ BinaryTreeReader(int columnId, Context context) throws IOException {
+ this(columnId, null, null, null, null, context);
}
protected BinaryTreeReader(int columnId, InStream present, InStream data, InStream length,
- OrcProto.ColumnEncoding encoding) throws IOException {
- super(columnId, present);
+ OrcProto.ColumnEncoding encoding, Context context) throws IOException {
+ super(columnId, present, context);
scratchlcv = new LongColumnVector();
this.stream = data;
if (length != null && encoding != null) {
checkEncoding(encoding);
- this.lengths = createIntegerReader(encoding.getKind(), length, false, false);
+ this.lengths = createIntegerReader(encoding.getKind(), length, false, context);
}
}
@@ -770,7 +817,7 @@ public class TreeReaderFactory {
OrcProto.Stream.Kind.DATA);
stream = streams.get(name);
lengths = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
- streams.get(new StreamName(columnId, OrcProto.Stream.Kind.LENGTH)), false, false);
+ streams.get(new StreamName(columnId, OrcProto.Stream.Kind.LENGTH)), false, context);
}
@Override
@@ -821,29 +868,33 @@ public class TreeReaderFactory {
private TimeZone writerTimeZone;
private boolean hasSameTZRules;
- TimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException {
- this(columnId, null, null, null, null, skipCorrupt);
+ TimestampTreeReader(int columnId, Context context) throws IOException {
+ this(columnId, null, null, null, null, context);
}
protected TimestampTreeReader(int columnId, InStream presentStream, InStream dataStream,
- InStream nanosStream, OrcProto.ColumnEncoding encoding, boolean skipCorrupt)
+ InStream nanosStream, OrcProto.ColumnEncoding encoding, Context context)
throws IOException {
- super(columnId, presentStream);
- this.skipCorrupt = skipCorrupt;
+ super(columnId, presentStream, context);
+ this.skipCorrupt = context.isSkipCorrupt();
this.baseTimestampMap = new HashMap<>();
this.readerTimeZone = TimeZone.getDefault();
- this.writerTimeZone = readerTimeZone;
+ if (context.getWriterTimezone() == null || context.getWriterTimezone().isEmpty()) {
+ this.writerTimeZone = readerTimeZone;
+ } else {
+ this.writerTimeZone = TimeZone.getTimeZone(context.getWriterTimezone());
+ }
this.hasSameTZRules = writerTimeZone.hasSameRules(readerTimeZone);
this.base_timestamp = getBaseTimestamp(readerTimeZone.getID());
if (encoding != null) {
checkEncoding(encoding);
if (dataStream != null) {
- this.data = createIntegerReader(encoding.getKind(), dataStream, true, skipCorrupt);
+ this.data = createIntegerReader(encoding.getKind(), dataStream, true, context);
}
if (nanosStream != null) {
- this.nanos = createIntegerReader(encoding.getKind(), nanosStream, false, skipCorrupt);
+ this.nanos = createIntegerReader(encoding.getKind(), nanosStream, false, context);
}
}
}
@@ -864,10 +915,10 @@ public class TreeReaderFactory {
super.startStripe(streams, stripeFooter);
data = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
streams.get(new StreamName(columnId,
- OrcProto.Stream.Kind.DATA)), true, skipCorrupt);
+ OrcProto.Stream.Kind.DATA)), true, context);
nanos = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
streams.get(new StreamName(columnId,
- OrcProto.Stream.Kind.SECONDARY)), false, skipCorrupt);
+ OrcProto.Stream.Kind.SECONDARY)), false, context);
base_timestamp = getBaseTimestamp(stripeFooter.getWriterTimezone());
}
@@ -973,16 +1024,16 @@ public class TreeReaderFactory {
public static class DateTreeReader extends TreeReader {
protected IntegerReader reader = null;
- DateTreeReader(int columnId) throws IOException {
- this(columnId, null, null, null);
+ DateTreeReader(int columnId, Context context) throws IOException {
+ this(columnId, null, null, null, context);
}
protected DateTreeReader(int columnId, InStream present, InStream data,
- OrcProto.ColumnEncoding encoding) throws IOException {
- super(columnId, present);
+ OrcProto.ColumnEncoding encoding, Context context) throws IOException {
+ super(columnId, present, context);
if (data != null && encoding != null) {
checkEncoding(encoding);
- reader = createIntegerReader(encoding.getKind(), data, true, false);
+ reader = createIntegerReader(encoding.getKind(), data, true, context);
}
}
@@ -1003,7 +1054,7 @@ public class TreeReaderFactory {
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
- streams.get(name), true, false);
+ streams.get(name), true, context);
}
@Override
@@ -1041,24 +1092,19 @@ public class TreeReaderFactory {
protected IntegerReader scaleReader = null;
private int[] scratchScaleVector;
- private final int precision;
- private final int scale;
-
- DecimalTreeReader(int columnId, int precision, int scale) throws IOException {
- this(columnId, precision, scale, null, null, null, null);
+ DecimalTreeReader(int columnId, Context context) throws IOException {
+ this(columnId, null, null, null, null, context);
}
- protected DecimalTreeReader(int columnId, int precision, int scale, InStream present,
- InStream valueStream, InStream scaleStream, OrcProto.ColumnEncoding encoding)
+ protected DecimalTreeReader(int columnId, InStream present,
+ InStream valueStream, InStream scaleStream, OrcProto.ColumnEncoding encoding, Context context)
throws IOException {
- super(columnId, present);
- this.precision = precision;
- this.scale = scale;
+ super(columnId, present, context);
this.scratchScaleVector = new int[VectorizedRowBatch.DEFAULT_SIZE];
this.valueStream = valueStream;
if (scaleStream != null && encoding != null) {
checkEncoding(encoding);
- this.scaleReader = createIntegerReader(encoding.getKind(), scaleStream, true, false);
+ this.scaleReader = createIntegerReader(encoding.getKind(), scaleStream, true, context);
}
}
@@ -1079,7 +1125,7 @@ public class TreeReaderFactory {
valueStream = streams.get(new StreamName(columnId,
OrcProto.Stream.Kind.DATA));
scaleReader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
- streams.get(new StreamName(columnId, OrcProto.Stream.Kind.SECONDARY)), true, false);
+ streams.get(new StreamName(columnId, OrcProto.Stream.Kind.SECONDARY)), true, context);
}
@Override
@@ -1143,13 +1189,13 @@ public class TreeReaderFactory {
public static class StringTreeReader extends TreeReader {
protected TreeReader reader;
- StringTreeReader(int columnId) throws IOException {
- super(columnId);
+ StringTreeReader(int columnId, Context context) throws IOException {
+ super(columnId, context);
}
protected StringTreeReader(int columnId, InStream present, InStream data, InStream length,
- InStream dictionary, OrcProto.ColumnEncoding encoding) throws IOException {
- super(columnId, present);
+ InStream dictionary, OrcProto.ColumnEncoding encoding, Context context) throws IOException {
+ super(columnId, present, context);
if (encoding != null) {
switch (encoding.getKind()) {
case DIRECT:
@@ -1160,7 +1206,7 @@ public class TreeReaderFactory {
case DICTIONARY:
case DICTIONARY_V2:
reader = new StringDictionaryTreeReader(columnId, present, data, length, dictionary,
- encoding);
+ encoding, context);
break;
default:
throw new IllegalArgumentException("Unsupported encoding " +
@@ -1187,7 +1233,7 @@ public class TreeReaderFactory {
break;
case DICTIONARY:
case DICTIONARY_V2:
- reader = new StringDictionaryTreeReader(columnId);
+ reader = new StringDictionaryTreeReader(columnId, context);
break;
default:
throw new IllegalArgumentException("Unsupported encoding " +
@@ -1312,11 +1358,11 @@ public class TreeReaderFactory {
protected StringDirectTreeReader(int columnId, InStream present, InStream data,
InStream length, OrcProto.ColumnEncoding.Kind encoding) throws IOException {
- super(columnId, present);
+ super(columnId, present, null);
this.scratchlcv = new LongColumnVector();
this.stream = data;
if (length != null && encoding != null) {
- this.lengths = createIntegerReader(encoding, length, false, false);
+ this.lengths = createIntegerReader(encoding, length, false, context);
this.data = SHIMS.getTextReaderShim(this.stream);
}
}
@@ -1341,7 +1387,7 @@ public class TreeReaderFactory {
data = SHIMS.getTextReaderShim(this.stream);
lengths = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
streams.get(new StreamName(columnId, OrcProto.Stream.Kind.LENGTH)),
- false, false);
+ false, context);
}
@Override
@@ -1406,17 +1452,17 @@ public class TreeReaderFactory {
private byte[] dictionaryBufferInBytesCache = null;
private final LongColumnVector scratchlcv;
- StringDictionaryTreeReader(int columnId) throws IOException {
- this(columnId, null, null, null, null, null);
+ StringDictionaryTreeReader(int columnId, Context context) throws IOException {
+ this(columnId, null, null, null, null, null, context);
}
protected StringDictionaryTreeReader(int columnId, InStream present, InStream data,
- InStream length, InStream dictionary, OrcProto.ColumnEncoding encoding)
- throws IOException {
- super(columnId, present);
+ InStream length, InStream dictionary, OrcProto.ColumnEncoding encoding,
+ Context context) throws IOException {
+ super(columnId, present, context);
scratchlcv = new LongColumnVector();
if (data != null && encoding != null) {
- this.reader = createIntegerReader(encoding.getKind(), data, false, false);
+ this.reader = createIntegerReader(encoding.getKind(), data, false, context);
}
if (dictionary != null && encoding != null) {
@@ -1457,14 +1503,14 @@ public class TreeReaderFactory {
// set up the row reader
name = new StreamName(columnId, OrcProto.Stream.Kind.DATA);
reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
- streams.get(name), false, false);
+ streams.get(name), false, context);
}
private void readDictionaryLengthStream(InStream in, OrcProto.ColumnEncoding encoding)
throws IOException {
int dictionarySize = encoding.getDictionarySize();
if (in != null) { // Guard against empty LENGTH stream.
- IntegerReader lenReader = createIntegerReader(encoding.getKind(), in, false, false);
+ IntegerReader lenReader = createIntegerReader(encoding.getKind(), in, false, context);
int offset = 0;
if (dictionaryOffsets == null ||
dictionaryOffsets.length < dictionarySize + 1) {
@@ -1599,7 +1645,7 @@ public class TreeReaderFactory {
protected CharTreeReader(int columnId, int maxLength, InStream present, InStream data,
InStream length, InStream dictionary, OrcProto.ColumnEncoding encoding) throws IOException {
- super(columnId, present, data, length, dictionary, encoding);
+ super(columnId, present, data, length, dictionary, encoding, null);
this.maxLength = maxLength;
}
@@ -1655,7 +1701,7 @@ public class TreeReaderFactory {
protected VarcharTreeReader(int columnId, int maxLength, InStream present, InStream data,
InStream length, InStream dictionary, OrcProto.ColumnEncoding encoding) throws IOException {
- super(columnId, present, data, length, dictionary, encoding);
+ super(columnId, present, data, length, dictionary, encoding, null);
this.maxLength = maxLength;
}
@@ -1706,16 +1752,14 @@ public class TreeReaderFactory {
protected StructTreeReader(int columnId,
TypeDescription readerSchema,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
- super(columnId);
+ Context context) throws IOException {
+ super(columnId, context);
List<TypeDescription> childrenTypes = readerSchema.getChildren();
this.fields = new TreeReader[childrenTypes.size()];
for (int i = 0; i < fields.length; ++i) {
TypeDescription subtype = childrenTypes.get(i);
- this.fields[i] = createTreeReader(subtype, evolution, included, skipCorrupt);
+ this.fields[i] = createTreeReader(subtype, context);
}
}
@@ -1791,16 +1835,14 @@ public class TreeReaderFactory {
protected UnionTreeReader(int fileColumn,
TypeDescription readerSchema,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
- super(fileColumn);
+ Context context) throws IOException {
+ super(fileColumn, context);
List<TypeDescription> childrenTypes = readerSchema.getChildren();
int fieldCount = childrenTypes.size();
this.fields = new TreeReader[fieldCount];
for (int i = 0; i < fieldCount; ++i) {
TypeDescription subtype = childrenTypes.get(i);
- this.fields[i] = createTreeReader(subtype, evolution, included, skipCorrupt);
+ this.fields[i] = createTreeReader(subtype, context);
}
}
@@ -1868,13 +1910,10 @@ public class TreeReaderFactory {
protected ListTreeReader(int fileColumn,
TypeDescription readerSchema,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
- super(fileColumn);
+ Context context) throws IOException {
+ super(fileColumn, context);
TypeDescription elementType = readerSchema.getChildren().get(0);
- elementReader = createTreeReader(elementType, evolution, included,
- skipCorrupt);
+ elementReader = createTreeReader(elementType, context);
}
@Override
@@ -1924,7 +1963,7 @@ public class TreeReaderFactory {
super.startStripe(streams, stripeFooter);
lengths = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
streams.get(new StreamName(columnId,
- OrcProto.Stream.Kind.LENGTH)), false, false);
+ OrcProto.Stream.Kind.LENGTH)), false, context);
if (elementReader != null) {
elementReader.startStripe(streams, stripeFooter);
}
@@ -1948,14 +1987,12 @@ public class TreeReaderFactory {
protected MapTreeReader(int fileColumn,
TypeDescription readerSchema,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt) throws IOException {
- super(fileColumn);
+ Context context) throws IOException {
+ super(fileColumn, context);
TypeDescription keyType = readerSchema.getChildren().get(0);
TypeDescription valueType = readerSchema.getChildren().get(1);
- keyReader = createTreeReader(keyType, evolution, included, skipCorrupt);
- valueReader = createTreeReader(valueType, evolution, included, skipCorrupt);
+ keyReader = createTreeReader(keyType, context);
+ valueReader = createTreeReader(valueType, context);
}
@Override
@@ -2007,7 +2044,7 @@ public class TreeReaderFactory {
super.startStripe(streams, stripeFooter);
lengths = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
streams.get(new StreamName(columnId,
- OrcProto.Stream.Kind.LENGTH)), false, false);
+ OrcProto.Stream.Kind.LENGTH)), false, context);
if (keyReader != null) {
keyReader.startStripe(streams, stripeFooter);
}
@@ -2029,10 +2066,10 @@ public class TreeReaderFactory {
}
public static TreeReader createTreeReader(TypeDescription readerType,
- SchemaEvolution evolution,
- boolean[] included,
- boolean skipCorrupt
+ Context context
) throws IOException {
+ final SchemaEvolution evolution = context.getSchemaEvolution();
+ final boolean[] included = evolution.getReaderIncluded();
TypeDescription fileType = evolution.getFileType(readerType);
if (fileType == null ||
(included != null && !included[readerType.getId()])) {
@@ -2045,8 +2082,7 @@ public class TreeReaderFactory {
readerTypeCategory != TypeDescription.Category.LIST &&
readerTypeCategory != TypeDescription.Category.UNION)) {
// We only convert complex children.
- return ConvertTreeReaderFactory.createConvertTreeReader(readerType, evolution,
- included, skipCorrupt);
+ return ConvertTreeReaderFactory.createConvertTreeReader(readerType, context);
}
switch (readerTypeCategory) {
case BOOLEAN:
@@ -2058,38 +2094,33 @@ public class TreeReaderFactory {
case FLOAT:
return new FloatTreeReader(fileType.getId());
case SHORT:
- return new ShortTreeReader(fileType.getId());
+ return new ShortTreeReader(fileType.getId(), context);
case INT:
- return new IntTreeReader(fileType.getId());
+ return new IntTreeReader(fileType.getId(), context);
case LONG:
- return new LongTreeReader(fileType.getId(), skipCorrupt);
+ return new LongTreeReader(fileType.getId(), context);
case STRING:
- return new StringTreeReader(fileType.getId());
+ return new StringTreeReader(fileType.getId(), context);
case CHAR:
return new CharTreeReader(fileType.getId(), readerType.getMaxLength());
case VARCHAR:
return new VarcharTreeReader(fileType.getId(), readerType.getMaxLength());
case BINARY:
- return new BinaryTreeReader(fileType.getId());
+ return new BinaryTreeReader(fileType.getId(), context);
case TIMESTAMP:
- return new TimestampTreeReader(fileType.getId(), skipCorrupt);
+ return new TimestampTreeReader(fileType.getId(), context);
case DATE:
- return new DateTreeReader(fileType.getId());
+ return new DateTreeReader(fileType.getId(), context);
case DECIMAL:
- return new DecimalTreeReader(fileType.getId(), readerType.getPrecision(),
- readerType.getScale());
+ return new DecimalTreeReader(fileType.getId(), context);
case STRUCT:
- return new StructTreeReader(fileType.getId(), readerType,
- evolution, included, skipCorrupt);
+ return new StructTreeReader(fileType.getId(), readerType, context);
case LIST:
- return new ListTreeReader(fileType.getId(), readerType,
- evolution, included, skipCorrupt);
+ return new ListTreeReader(fileType.getId(), readerType, context);
case MAP:
- return new MapTreeReader(fileType.getId(), readerType, evolution,
- included, skipCorrupt);
+ return new MapTreeReader(fileType.getId(), readerType, context);
case UNION:
- return new UnionTreeReader(fileType.getId(), readerType,
- evolution, included, skipCorrupt);
+ return new UnionTreeReader(fileType.getId(), readerType, context);
default:
throw new IllegalArgumentException("Unsupported type " +
readerTypeCategory);