You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2020/08/21 18:02:21 UTC
[orc] branch branch-1.6 updated: ORC-644: Support positional
mapping for nested types.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-1.6
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/branch-1.6 by this push:
new 4e7ea77 ORC-644: Support positional mapping for nested types.
4e7ea77 is described below
commit 4e7ea77a2cdbfa58d0b475bdae53a7aa945d40b7
Author: Arvin Zheng <zm...@gmail.com>
AuthorDate: Mon Jul 27 17:32:04 2020 -0700
ORC-644: Support positional mapping for nested types.
Resolves #522
Signed-off-by: Owen O'Malley <om...@apache.org>
---
java/core/src/java/org/apache/orc/Reader.java | 4 +-
.../java/org/apache/orc/impl/SchemaEvolution.java | 2 +-
.../org/apache/orc/impl/TestSchemaEvolution.java | 43 ++++++++++++++++++++++
3 files changed, 46 insertions(+), 3 deletions(-)
diff --git a/java/core/src/java/org/apache/orc/Reader.java b/java/core/src/java/org/apache/orc/Reader.java
index 69a01de..e918c00 100644
--- a/java/core/src/java/org/apache/orc/Reader.java
+++ b/java/core/src/java/org/apache/orc/Reader.java
@@ -300,9 +300,9 @@ public interface Reader extends Closeable {
}
/**
- * Set no. of levels to force schema evolution to be positional instead of
+ * Set number of levels to force schema evolution to be positional instead of
* based on the column names.
- * @param value force positional evolution
+ * @param value number of levels of positional schema evolution
* @return this
*/
public Options positionalEvolutionLevel(int value) {
diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
index dac5437..d7c6bd2 100644
--- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
+++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
@@ -497,7 +497,7 @@ public class SchemaEvolution {
if (fileChildren.size() == readerChildren.size()) {
for(int i=0; i < fileChildren.size(); ++i) {
buildConversion(fileChildren.get(i),
- readerChildren.get(i), 0);
+ readerChildren.get(i), positionalLevels - 1);
}
} else {
isOk = false;
diff --git a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
index 37d326e..6f9b0d6 100644
--- a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
+++ b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
@@ -1732,6 +1732,49 @@ public class TestSchemaEvolution {
assertEquals(4, evo.getFileType(4).getId());
}
+ @Test
+ public void testPositionalEvolutionForStructInArray() throws IOException {
+ options.forcePositionalEvolution(true);
+ options.positionalEvolutionLevel(Integer.MAX_VALUE);
+ TypeDescription file = TypeDescription.fromString("array<struct<x:int,y:int,z:int>>");
+ TypeDescription read = TypeDescription.fromString("array<struct<z:int,x:int,a:int,b:int>>");
+ SchemaEvolution evo = new SchemaEvolution(file, read, options);
+ assertEquals(1, evo.getFileType(1).getId());
+ assertEquals(2, evo.getFileType(2).getId());
+ assertEquals(3, evo.getFileType(3).getId());
+ assertEquals(4, evo.getFileType(4).getId());
+ assertEquals(null, evo.getFileType(5));
+ }
+
+ @Test
+ public void testPositionalEvolutionForTwoLayerNestedStruct() throws IOException {
+ options.forcePositionalEvolution(true);
+ options.positionalEvolutionLevel(Integer.MAX_VALUE);
+ TypeDescription file = TypeDescription.fromString("struct<s:struct<x:int,y:int,z:int>>");
+ TypeDescription read = TypeDescription.fromString("struct<s:struct<z:int,x:int,a:int,b:int>>");
+ SchemaEvolution evo = new SchemaEvolution(file, read, options);
+ assertEquals(1, evo.getFileType(1).getId());
+ assertEquals(2, evo.getFileType(2).getId());
+ assertEquals(3, evo.getFileType(3).getId());
+ assertEquals(4, evo.getFileType(4).getId());
+ assertNull(evo.getFileType(5));
+ }
+
+ @Test
+ public void testPositionalEvolutionForThreeLayerNestedStruct() throws IOException {
+ options.forcePositionalEvolution(true);
+ options.positionalEvolutionLevel(Integer.MAX_VALUE);
+ TypeDescription file = TypeDescription.fromString("struct<s1:struct<s2:struct<x:int,y:int,z:int>>>");
+ TypeDescription read = TypeDescription.fromString("struct<s1:struct<s:struct<z:int,x:int,a:int,b:int>>>");
+ SchemaEvolution evo = new SchemaEvolution(file, read, options);
+ assertEquals(1, evo.getFileType(1).getId());
+ assertEquals(2, evo.getFileType(2).getId());
+ assertEquals(3, evo.getFileType(3).getId());
+ assertEquals(4, evo.getFileType(4).getId());
+ assertEquals(5, evo.getFileType(5).getId());
+ assertNull(evo.getFileType(6));
+ }
+
// These are helper methods that pull some of the common code into one
// place.