You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2016/07/01 17:56:21 UTC
[6/8] orc git commit: HIVE-13648. ORC schema evolution doesn't
support the same type conversions for varchar, char,
or decimal when max length, precision, or scale are different.
HIVE-13648. ORC schema evolution doesn't support the same type conversions for
varchar, char, or decimal when max length, precision, or scale are different.
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/775bc2ad
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/775bc2ad
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/775bc2ad
Branch: refs/heads/branch-1.1
Commit: 775bc2ad91f83fbf1e99d1af10567767be1119af
Parents: 6b61f42
Author: Owen O'Malley <om...@apache.org>
Authored: Thu Jun 30 10:26:24 2016 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Jul 1 09:33:08 2016 -0700
----------------------------------------------------------------------
.../orc/impl/ConvertTreeReaderFactory.java | 28 +++++++++++---------
.../org/apache/orc/impl/SchemaEvolution.java | 6 ++---
.../org/apache/orc/impl/TreeReaderFactory.java | 2 +-
3 files changed, 18 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/775bc2ad/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index 753e5bc..eda47d3 100644
--- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -257,7 +257,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
bytesColVector.vector[elementNum],
bytesColVector.start[elementNum], bytesColVector.length[elementNum],
StandardCharsets.UTF_8);
-
+
return string;
}
@@ -1323,6 +1323,8 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private DecimalTreeReader decimalTreeReader;
+ private final TypeDescription fileType;
+ private final TypeDescription readerType;
private DecimalColumnVector fileDecimalColVector;
private int filePrecision;
private int fileScale;
@@ -1333,8 +1335,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
DecimalFromDecimalTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType)
throws IOException {
super(columnId);
+ this.fileType = fileType;
filePrecision = fileType.getPrecision();
fileScale = fileType.getScale();
+ this.readerType = readerType;
readerPrecision = readerType.getPrecision();
readerScale = readerType.getScale();
decimalTreeReader = new DecimalTreeReader(columnId, filePrecision, fileScale);
@@ -2248,7 +2252,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
return new TimestampFromDecimalTreeReader(columnId, fileType, skipCorrupt);
case DECIMAL:
- // UNDONE: Decimal to Decimal conversion????
+ return new DecimalFromDecimalTreeReader(columnId, fileType, readerType);
// Not currently supported conversion(s):
case BINARY:
@@ -2354,8 +2358,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
case CHAR:
- throw new IllegalArgumentException("No conversion of type " +
- readerType.getCategory() + " to self needed");
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
case BINARY:
return new BinaryFromStringGroupTreeReader(columnId, fileType);
@@ -2411,8 +2414,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
case VARCHAR:
- throw new IllegalArgumentException("No conversion of type " +
- readerType.getCategory() + " to self needed");
+ return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
case BINARY:
return new BinaryFromStringGroupTreeReader(columnId, fileType);
@@ -2628,11 +2630,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
* StringGroupFromFloatTreeReader (written)
* StringGroupFromDoubleTreeReader (written)
* StringGroupFromDecimalTreeReader (written)
- *
+ *
* String from Char/Varchar conversion
* Char from String/Varchar conversion
* Varchar from String/Char conversion
- *
+ *
* StringGroupFromTimestampTreeReader (written)
* StringGroupFromDateTreeReader (written)
* StringGroupFromBinaryTreeReader *****
@@ -2650,7 +2652,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
* TimestampFromDecimalTreeeReader (written)
* TimestampFromStringGroupTreeReader (written)
* TimestampFromDateTreeReader
- *
+ *
*
* To DATE:
* Convert from (STRING, CHAR, VARCHAR) using string conversion.
@@ -2780,7 +2782,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
// Fall through.
}
- // Now look for the few cases we don't convert from
+ // Now look for the few cases we don't convert from
switch (fileType.getCategory()) {
case BOOLEAN:
@@ -2799,8 +2801,8 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
default:
return true;
}
-
-
+
+
case STRING:
case CHAR:
case VARCHAR:
@@ -2836,7 +2838,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
default:
return true;
}
-
+
case BINARY:
switch (readerType.getCategory()) {
// Not currently supported conversion(s):
http://git-wip-us.apache.org/repos/asf/orc/blob/775bc2ad/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
index 68000d6..07b527d 100644
--- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
+++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
@@ -100,12 +100,10 @@ public class SchemaEvolution {
break;
case CHAR:
case VARCHAR:
- // HIVE-13648: Look at ORC data type conversion edge cases (CHAR, VARCHAR, DECIMAL)
- isOk = fileType.getMaxLength() == readerType.getMaxLength();
+ // We do conversion when same CHAR/VARCHAR type but different maxLength.
break;
case DECIMAL:
- // HIVE-13648: Look at ORC data type conversion edge cases (CHAR, VARCHAR, DECIMAL)
- // TODO we don't enforce scale and precision checks, but probably should
+ // We do conversion when same DECIMAL type but different precision/scale.
break;
case UNION:
case MAP:
http://git-wip-us.apache.org/repos/asf/orc/blob/775bc2ad/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
index d390ff4..3371e1e 100644
--- a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
@@ -2036,7 +2036,7 @@ public class TreeReaderFactory {
return new NullTreeReader(0);
}
TypeDescription.Category readerTypeCategory = readerType.getCategory();
- if (!fileType.getCategory().equals(readerTypeCategory) &&
+ if (!fileType.equals(readerType) &&
(readerTypeCategory != TypeDescription.Category.STRUCT &&
readerTypeCategory != TypeDescription.Category.MAP &&
readerTypeCategory != TypeDescription.Category.LIST &&