You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2016/09/03 02:55:45 UTC
hive git commit: HIVE-14607: ORC split generation failed with
exception: java.lang.ArrayIndexOutOfBoundsException: 1 (Matt McCline,
Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master ec22d1c66 -> 8d7dbd49e
HIVE-14607: ORC split generation failed with exception: java.lang.ArrayIndexOutOfBoundsException: 1 (Matt McCline, Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8d7dbd49
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8d7dbd49
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8d7dbd49
Branch: refs/heads/master
Commit: 8d7dbd49ece51c8405c057041a9fc5326bfee494
Parents: ec22d1c
Author: Matt McCline <mm...@hortonworks.com>
Authored: Fri Sep 2 19:55:34 2016 -0700
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Fri Sep 2 19:55:34 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 21 +++++++-----
.../ql/TestTxnCommands2WithSplitUpdate.java | 36 --------------------
2 files changed, 13 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8d7dbd49/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 70003ed..ccb39da 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -1162,7 +1162,9 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
private List<StripeInformation> stripes;
private List<StripeStatistics> stripeStats;
private List<OrcProto.Type> fileTypes;
- private boolean[] readerIncluded;
+ private boolean[] included; // The included columns from the Hive configuration.
+ private boolean[] readerIncluded; // The included columns of the reader / file schema that
+ // include ACID columns if present.
private final boolean isOriginal;
private final List<DeltaMetaData> deltas;
private final boolean hasBase;
@@ -1357,7 +1359,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
if ((deltas == null || deltas.isEmpty()) && context.sarg != null) {
String[] colNames =
extractNeededColNames((readerTypes == null ? fileTypes : readerTypes),
- context.conf, readerIncluded, isOriginal);
+ context.conf, included, isOriginal);
if (colNames == null) {
LOG.warn("Skipping split elimination for {} as column names is null", file.getPath());
} else {
@@ -1479,15 +1481,18 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
fileTypes = orcTail.getTypes();
TypeDescription fileSchema = OrcUtils.convertTypeFromProtobuf(fileTypes, 0);
if (readerTypes == null) {
- readerIncluded = genIncludedColumns(fileTypes, context.conf, isOriginal);
- evolution = new SchemaEvolution(fileSchema, readerIncluded);
+ included = genIncludedColumns(fileTypes, context.conf, isOriginal);
+ evolution = new SchemaEvolution(fileSchema, included);
+ readerIncluded = included;
} else {
- // The readerSchema always comes in without ACID columns.
- readerIncluded = genIncludedColumns(readerTypes, context.conf, /* isOriginal */ true);
- if (readerIncluded != null && !isOriginal) {
+ // The reader schema always comes in without ACID columns.
+ included = genIncludedColumns(readerTypes, context.conf, /* isOriginal */ true);
+ if (included != null && !isOriginal) {
// We shift the include columns here because the SchemaEvolution constructor will
// add the ACID event metadata the readerSchema...
- readerIncluded = shiftReaderIncludedForAcid(readerIncluded);
+ readerIncluded = shiftReaderIncludedForAcid(included);
+ } else {
+ readerIncluded = included;
}
TypeDescription readerSchema = OrcUtils.convertTypeFromProtobuf(readerTypes, 0);
evolution = new SchemaEvolution(fileSchema, readerSchema, readerIncluded);
http://git-wip-us.apache.org/repos/asf/hive/blob/8d7dbd49/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
index becb22a..c2330cb 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
@@ -60,42 +60,6 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 {
@Override
@Test
- public void testOrcPPD() throws Exception {
- final String defaultUnset = "unset";
- String oldSplitStrategyValue = hiveConf.get(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, defaultUnset);
- // TODO: Setting split strategy as 'BI' is workaround for HIVE-14448 until it is resolved.
- hiveConf.set(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, "BI");
-
- super.testOrcPPD();
-
- // Restore the previous value for split strategy, or unset if not previously set.
- if (oldSplitStrategyValue.equals(defaultUnset)) {
- hiveConf.unset(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname);
- } else {
- hiveConf.set(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, oldSplitStrategyValue);
- }
- }
-
- @Override
- @Test
- public void testOrcNoPPD() throws Exception {
- final String defaultUnset = "unset";
- String oldSplitStrategyValue = hiveConf.get(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, defaultUnset);
- // TODO: Setting split strategy as 'BI' is workaround for HIVE-14448 until it is resolved.
- hiveConf.set(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, "BI");
-
- super.testOrcNoPPD();
-
- // Restore the previous value for split strategy, or unset if not previously set.
- if (oldSplitStrategyValue.equals(defaultUnset)) {
- hiveConf.unset(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname);
- } else {
- hiveConf.set(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, oldSplitStrategyValue);
- }
- }
-
- @Override
- @Test
public void testInitiatorWithMultipleFailedCompactions() throws Exception {
// Test with split-update turned on.
testInitiatorWithMultipleFailedCompactionsForVariousTblProperties("'transactional'='true','transactional_properties'='default'");