You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2017/01/31 16:49:20 UTC
orc git commit: ORC-138. Fix part of HIVE-15335 that changed decimal
schema evolution. (omalley)
Repository: orc
Updated Branches:
refs/heads/master d33200e3e -> f3c3fe81f
ORC-138. Fix part of HIVE-15335 that changed decimal schema evolution.
(omalley)
Fixes #88
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/f3c3fe81
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/f3c3fe81
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/f3c3fe81
Branch: refs/heads/master
Commit: f3c3fe81fc9ba5641bdf18858db5d175495b2e80
Parents: d33200e
Author: Owen O'Malley <om...@apache.org>
Authored: Fri Jan 27 16:07:34 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Jan 31 08:48:02 2017 -0800
----------------------------------------------------------------------
.../orc/impl/ConvertTreeReaderFactory.java | 64 +++++++++++++++++---
1 file changed, 54 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/f3c3fe81/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index ae43824..7964340 100644
--- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -599,18 +599,55 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
setConvertTreeReader(decimalTreeReader);
}
- private static HiveDecimal DECIMAL_MAX_LONG = HiveDecimal.create(Long.MAX_VALUE);
- private static HiveDecimal DECIMAL_MIN_LONG = HiveDecimal.create(Long.MIN_VALUE);
-
@Override
public void setConvertVectorElement(int elementNum) throws IOException {
- HiveDecimal decimalValue = decimalColVector.vector[elementNum].getHiveDecimal();
- if (decimalValue.compareTo(DECIMAL_MAX_LONG) > 0 ||
- decimalValue.compareTo(DECIMAL_MIN_LONG) < 0) {
+ HiveDecimalWritable decWritable = decimalColVector.vector[elementNum];
+ long[] vector = longColVector.vector;
+ Category readerCategory = readerType.getCategory();
+
+ // Check to see if the decimal will fit in the Hive integer data type.
+ // If not, set the element to null.
+ boolean isInRange;
+ switch (readerCategory) {
+ case BOOLEAN:
+ // No data loss for boolean.
+ vector[elementNum] = decWritable.signum() == 0 ? 0 : 1;
+ return;
+ case BYTE:
+ isInRange = decWritable.isByte();
+ break;
+ case SHORT:
+ isInRange = decWritable.isShort();
+ break;
+ case INT:
+ isInRange = decWritable.isInt();
+ break;
+ case LONG:
+ isInRange = decWritable.isLong();
+ break;
+ default:
+ throw new RuntimeException("Unexpected type kind " + readerCategory.name());
+ }
+ if (!isInRange) {
longColVector.isNull[elementNum] = true;
longColVector.noNulls = false;
} else {
- downCastAnyInteger(longColVector, elementNum, decimalValue.longValue(), readerType);
+ switch (readerCategory) {
+ case BYTE:
+ vector[elementNum] = decWritable.byteValue();
+ break;
+ case SHORT:
+ vector[elementNum] = decWritable.shortValue();
+ break;
+ case INT:
+ vector[elementNum] = decWritable.intValue();
+ break;
+ case LONG:
+ vector[elementNum] = decWritable.longValue();
+ break;
+ default:
+ throw new RuntimeException("Unexpected type kind " + readerCategory.name());
+ }
}
}
@@ -1520,6 +1557,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private final TypeDescription readerType;
private DecimalColumnVector decimalColVector;
private BytesColumnVector bytesColVector;
+ private byte[] scratchBuffer;
StringGroupFromDecimalTreeReader(int columnId, TypeDescription fileType,
TypeDescription readerType, Context context) throws IOException {
@@ -1529,13 +1567,19 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
this.readerType = readerType;
decimalTreeReader = new DecimalTreeReader(columnId, context);
setConvertTreeReader(decimalTreeReader);
+ scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES];
}
@Override
public void setConvertVectorElement(int elementNum) {
- String string = decimalColVector.vector[elementNum].getHiveDecimal().toString();
- byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
- assignStringGroupVectorEntry(bytesColVector, elementNum, readerType, bytes);
+ HiveDecimalWritable decWritable = decimalColVector.vector[elementNum];
+
+ // Convert decimal into bytes instead of a String for better performance.
+ final int byteIndex = decWritable.toBytes(scratchBuffer);
+
+ assignStringGroupVectorEntry(
+ bytesColVector, elementNum, readerType,
+ scratchBuffer, byteIndex, HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES - byteIndex);
}
@Override