You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2018/05/07 18:19:35 UTC
[1/2] orc git commit: ORC-360. Add Java error checking on the types
when reading.
Repository: orc
Updated Branches:
refs/heads/branch-1.4 7910fa968 -> ad33a70b1
ORC-360. Add Java error checking on the types when reading.
Fixes #266
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/d5018d30
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/d5018d30
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/d5018d30
Branch: refs/heads/branch-1.4
Commit: d5018d309a8adc6b8e0567cb692a17371d16e108
Parents: 7910fa9
Author: Owen O'Malley <om...@apache.org>
Authored: Mon May 7 09:19:17 2018 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Mon May 7 10:40:55 2018 -0700
----------------------------------------------------------------------
java/core/src/java/org/apache/orc/OrcUtils.java | 56 +++++++++++++++++++-
.../java/org/apache/orc/impl/ReaderImpl.java | 1 +
2 files changed, 55 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/d5018d30/java/core/src/java/org/apache/orc/OrcUtils.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/OrcUtils.java b/java/core/src/java/org/apache/orc/OrcUtils.java
index 37e624c..361a177 100644
--- a/java/core/src/java/org/apache/orc/OrcUtils.java
+++ b/java/core/src/java/org/apache/orc/OrcUtils.java
@@ -19,6 +19,7 @@ package org.apache.orc;
import org.apache.orc.impl.ReaderImpl;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -245,7 +246,7 @@ public class OrcUtils {
{
// Make room for MAP type.
result.add(null);
-
+
// Add MAP type pair in order to determine their subtype values.
appendOrcTypesRebuildSubtypes(result, children.get(0));
int subtype2 = result.size();
@@ -383,7 +384,7 @@ public class OrcUtils {
{
// Make room for MAP type.
result.add(null);
-
+
// Add MAP type pair in order to determine their subtype values.
columnId = appendOrcTypesRebuildSubtypes(result, types, columnId);
int subtype2 = result.size();
@@ -451,6 +452,57 @@ public class OrcUtils {
}
/**
+ * Checks whether the list of protobuf types from the file are valid or not.
+ * @param types the list of types from the protobuf
+ * @param root the top of the tree to check
+ * @return the next available id
+ * @throws java.io.IOException if the tree is invalid
+ */
+ public static int isValidTypeTree(List<OrcProto.Type> types,
+ int root) throws IOException {
+ if (root < 0 || root >= types.size()) {
+ throw new IOException("Illegal type id " + root +
+ ". The valid range is 0 to " + (types.size() - 1));
+ }
+ OrcProto.Type rootType = types.get(root);
+ int current = root+1;
+ List<Integer> children = rootType.getSubtypesList();
+ if (!rootType.hasKind()) {
+ throw new IOException("Type " + root + " has an unknown kind.");
+ }
+ // ensure that we have the right number of children
+ switch(rootType.getKind()) {
+ case LIST:
+ if (children == null || children.size() != 1) {
+ throw new IOException("Wrong number of type children in list " + root);
+ }
+ break;
+ case MAP:
+ if (children == null || children.size() != 2) {
+ throw new IOException("Wrong number of type children in map " + root);
+ }
+ break;
+ case UNION:
+ case STRUCT:
+ break;
+ default:
+ if (children != null && children.size() != 0) {
+ throw new IOException("Type children under primitive type " + root);
+ }
+ }
+ // ensure the children are also correct
+ if (children != null) {
+ for(int child: children) {
+ if (child != current) {
+ throw new IOException("Unexpected child type id " + child + " when " +
+ current + " was expected.");
+ }
+ current = isValidTypeTree(types, current);
+ }
+ }
+ return current;
+ }
+ /**
* Translate the given rootColumn from the list of types to a TypeDescription.
* @param types all of the types
* @param rootColumn translate this type
http://git-wip-us.apache.org/repos/asf/orc/blob/d5018d30/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
index 9146b23..80b7a60 100644
--- a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
@@ -381,6 +381,7 @@ public class ReaderImpl implements Reader {
this.stripes = tail.getStripes();
this.stripeStats = tail.getStripeStatisticsProto();
}
+ OrcUtils.isValidTypeTree(this.types, 0);
this.schema = OrcUtils.convertTypeFromProtobuf(this.types, 0);
}
[2/2] orc git commit: Preparing for ORC 1.4.4 release.
Posted by om...@apache.org.
Preparing for ORC 1.4.4 release.
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/ad33a70b
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/ad33a70b
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/ad33a70b
Branch: refs/heads/branch-1.4
Commit: ad33a70b14ad9073cba9cd12087ffa31a6403f6f
Parents: d5018d3
Author: Owen O'Malley <om...@apache.org>
Authored: Mon May 7 11:19:17 2018 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Mon May 7 11:19:17 2018 -0700
----------------------------------------------------------------------
CMakeLists.txt | 2 +-
java/core/pom.xml | 2 +-
java/mapreduce/pom.xml | 2 +-
java/pom.xml | 8 ++++----
java/tools/pom.xml | 2 +-
5 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/ad33a70b/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1c04819..ac55fed 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -17,7 +17,7 @@ project(ORC)
# Version number of package
SET(CPACK_PACKAGE_VERSION_MAJOR "1")
SET(CPACK_PACKAGE_VERSION_MINOR "4")
-SET(CPACK_PACKAGE_VERSION_PATCH "4-SNAPSHOT")
+SET(CPACK_PACKAGE_VERSION_PATCH "4")
SET(ORC_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}")
option (BUILD_JAVA
http://git-wip-us.apache.org/repos/asf/orc/blob/ad33a70b/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/java/core/pom.xml b/java/core/pom.xml
index d767962..87effe2 100644
--- a/java/core/pom.xml
+++ b/java/core/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.orc</groupId>
<artifactId>orc</artifactId>
- <version>1.4.4-SNAPSHOT</version>
+ <version>1.4.4</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/orc/blob/ad33a70b/java/mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/java/mapreduce/pom.xml b/java/mapreduce/pom.xml
index 05ca586..724cf90 100644
--- a/java/mapreduce/pom.xml
+++ b/java/mapreduce/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.orc</groupId>
<artifactId>orc</artifactId>
- <version>1.4.4-SNAPSHOT</version>
+ <version>1.4.4</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/orc/blob/ad33a70b/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index f6d29fd..0a041c3 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -21,7 +21,7 @@
</parent>
<groupId>org.apache.orc</groupId>
<artifactId>orc</artifactId>
- <version>1.4.4-SNAPSHOT</version>
+ <version>1.4.4</version>
<packaging>pom</packaging>
<name>Apache ORC</name>
@@ -313,17 +313,17 @@
<dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
- <version>1.4.4-SNAPSHOT</version>
+ <version>1.4.4</version>
</dependency>
<dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-mapreduce</artifactId>
- <version>1.4.4-SNAPSHOT</version>
+ <version>1.4.4</version>
</dependency>
<dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-tools</artifactId>
- <version>1.4.4-SNAPSHOT</version>
+ <version>1.4.4</version>
</dependency>
<!-- inter-project depedencies -->
http://git-wip-us.apache.org/repos/asf/orc/blob/ad33a70b/java/tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index cd9eb8c..beb048e 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.orc</groupId>
<artifactId>orc</artifactId>
- <version>1.4.4-SNAPSHOT</version>
+ <version>1.4.4</version>
<relativePath>../pom.xml</relativePath>
</parent>