You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by do...@apache.org on 2021/12/15 23:36:48 UTC
[orc] branch main updated: ORC-1059: Align findColumns behaviour between 1.6 and 1.7 release (#972)
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/main by this push:
new aee8132 ORC-1059: Align findColumns behaviour between 1.6 and 1.7 release (#972)
aee8132 is described below
commit aee81324507de1c5eb8e86bd7aa93a0f6b02bfd9
Author: Panagiotis Garefalakis <pg...@apache.org>
AuthorDate: Thu Dec 16 01:35:50 2021 +0200
ORC-1059: Align findColumns behaviour between 1.6 and 1.7 release (#972)
### What changes were proposed in this pull request?
When trying to bump Apache Hive to 1.7 I noticed query failures related to SArgs pushdown.
In more detail, Hive may push Partition column filters that are not part of the columns, until 1.6 recently these columns were ignored, while 1.7 throws IllegalArgumentException.
### Why are the changes needed?
Align findColumns behaviour between 1.6 and 1.7 release
### How was this patch tested?
Existing tests
---
.../src/java/org/apache/orc/impl/RecordReaderImpl.java | 10 ++++++++--
.../test/org/apache/orc/mapred/TestOrcFileEvolution.java | 16 +++++++---------
2 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
index aade2ef..d9bbf44 100644
--- a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
@@ -184,8 +184,14 @@ public class RecordReaderImpl implements RecordReader {
SchemaEvolution evolution) {
int[] result = new int[sargLeaves.size()];
for (int i = 0; i < sargLeaves.size(); ++i) {
- String colName = sargLeaves.get(i).getColumnName();
- result[i] = findColumns(evolution, colName);
+ int colNum = -1;
+ try {
+ String colName = sargLeaves.get(i).getColumnName();
+ colNum = findColumns(evolution, colName);
+ } catch (IllegalArgumentException e) {
+ LOG.debug("{}", e.getMessage());
+ }
+ result[i] = colNum;
}
return result;
}
diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcFileEvolution.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcFileEvolution.java
index 3f876ea..d8102ed 100644
--- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcFileEvolution.java
+++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcFileEvolution.java
@@ -237,15 +237,13 @@ public class TestOrcFileEvolution {
@Test
public void testMissingColumnFromReaderSchema() {
- // Expect failure if the column is missing from the reader schema, as column a that is added
- // by the SArg is missing from the reader schema
- assertThrows(IllegalArgumentException.class, () -> {
- checkEvolution("struct<b:int,c:string>",
- "struct<b:int,c:string>",
- struct(1, "foo"),
- struct(1, "foo", null),
- true, true, false);
- }, "Field a not found in");
+ // If column is part of the SArg but is missing from the reader schema
+ // will be ignored (consistent with 1.6 release behaviour)
+ checkEvolution("struct<b:int,c:string>",
+ "struct<b:int,c:string>",
+ struct(1, "foo"),
+ struct(1, "foo", null),
+ true, true, false);
}
@ParameterizedTest