You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2016/12/14 16:25:36 UTC
orc git commit: ORC-120. Add option to force positional matching of
schema evolution.
Repository: orc
Updated Branches:
refs/heads/master ee1ead769 -> db00ffdb9
ORC-120. Add option to force positional matching of schema evolution.
Fixes #72
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/db00ffdb
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/db00ffdb
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/db00ffdb
Branch: refs/heads/master
Commit: db00ffdb9f8af5fc0dd163d7170eab003ae81493
Parents: ee1ead7
Author: Owen O'Malley <om...@apache.org>
Authored: Mon Dec 12 18:49:44 2016 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Wed Dec 14 08:25:13 2016 -0800
----------------------------------------------------------------------
java/core/src/java/org/apache/orc/OrcConf.java | 7 ++++++-
java/core/src/java/org/apache/orc/Reader.java | 17 +++++++++++++++++
.../java/org/apache/orc/impl/SchemaEvolution.java | 5 ++++-
.../org/apache/orc/impl/TestSchemaEvolution.java | 12 ++++++++++++
4 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/db00ffdb/java/core/src/java/org/apache/orc/OrcConf.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/OrcConf.java b/java/core/src/java/org/apache/orc/OrcConf.java
index 05ab13b..5eb2aa3 100644
--- a/java/core/src/java/org/apache/orc/OrcConf.java
+++ b/java/core/src/java/org/apache/orc/OrcConf.java
@@ -137,7 +137,12 @@ public enum OrcConf {
KRYO_SARG("orc.kryo.sarg", "orc.kryo.sarg", null,
"The kryo and base64 encoded SearchArgument for predicate pushdown."),
SARG_COLUMNS("orc.sarg.column.names", "org.sarg.column.names", null,
- "The list of column names for the SearchArgument.")
+ "The list of column names for the SearchArgument."),
+ FORCE_POSITIONAL_EVOLUTION("orc.force.positional.evolution",
+ "orc.force.positional.evolution", false,
+ "Require schema evolution to match the top level columns using position\n" +
+ "rather than column names. This provides backwards compatibility with\n" +
+ "Hive 2.1.")
;
private final String attribute;
http://git-wip-us.apache.org/repos/asf/orc/blob/db00ffdb/java/core/src/java/org/apache/orc/Reader.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/Reader.java b/java/core/src/java/org/apache/orc/Reader.java
index c0e6109..3c9681c 100644
--- a/java/core/src/java/org/apache/orc/Reader.java
+++ b/java/core/src/java/org/apache/orc/Reader.java
@@ -159,6 +159,7 @@ public interface Reader {
private TypeDescription schema = null;
private DataReader dataReader = null;
private Boolean tolerateMissingSchema = null;
+ private boolean forcePositionalEvolution;
public Options() {
// PASS
@@ -168,6 +169,7 @@ public interface Reader {
useZeroCopy = OrcConf.USE_ZEROCOPY.getBoolean(conf);
skipCorruptRecords = OrcConf.SKIP_CORRUPT_DATA.getBoolean(conf);
tolerateMissingSchema = OrcConf.TOLERATE_MISSING_SCHEMA.getBoolean(conf);
+ forcePositionalEvolution = OrcConf.FORCE_POSITIONAL_EVOLUTION.getBoolean(conf);
}
/**
@@ -249,6 +251,17 @@ public interface Reader {
return this;
}
+ /**
+ * Set whether to force schema evolution to be positional instead of
+ * based on the column names.
+ * @param value force positional evolution
+ * @return this
+ */
+ public Options forcePositionalEvolution(boolean value) {
+ this.forcePositionalEvolution = value;
+ return this;
+ }
+
public boolean[] getInclude() {
return include;
}
@@ -293,6 +306,10 @@ public interface Reader {
return dataReader;
}
+ public boolean getForcePositionalEvolution() {
+ return forcePositionalEvolution;
+ }
+
public Options clone() {
Options result = new Options();
result.include = include;
http://git-wip-us.apache.org/repos/asf/orc/blob/db00ffdb/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
index b76a319..f6f3ac7 100644
--- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
+++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
@@ -95,7 +95,10 @@ public class SchemaEvolution {
this.readerFileTypes =
new TypeDescription[this.readerSchema.getMaximumId() + 1];
int positionalLevels = 0;
- if (!hasColumnNames(isAcid? getBaseRow(fileSchema) : fileSchema)){
+ if (options.getForcePositionalEvolution()) {
+ positionalLevels = isAcid ? 2 : 1;
+ buildConversion(fileSchema, this.readerSchema, positionalLevels);
+ } else if (!hasColumnNames(isAcid? getBaseRow(fileSchema) : fileSchema)) {
if (!this.fileSchema.equals(this.readerSchema)) {
if (!allowMissingMetadata) {
throw new RuntimeException("Found that schema metadata is missing"
http://git-wip-us.apache.org/repos/asf/orc/blob/db00ffdb/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
index c542711..ac0115d 100644
--- a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
+++ b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
@@ -1073,4 +1073,16 @@ public class TestSchemaEvolution {
((LongColumnVector) batch.cols[2]).vector[r]);
}
}
+
+ @Test
+ public void testPositionalEvolution() throws IOException {
+ options.forcePositionalEvolution(true);
+ TypeDescription file = TypeDescription.fromString("struct<x:int,y:int,z:int>");
+ TypeDescription read = TypeDescription.fromString("struct<z:int,x:int,a:int,b:int>");
+ SchemaEvolution evo = new SchemaEvolution(file, read, options);
+ assertEquals(1, evo.getFileType(1).getId());
+ assertEquals(2, evo.getFileType(2).getId());
+ assertEquals(3, evo.getFileType(3).getId());
+ assertEquals(null, evo.getFileType(4));
+ }
}