You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2019/04/16 19:52:13 UTC
[hive] branch branch-2.3 updated: HIVE-21585: Upgrade Hive
branch-2.3 to ORC.1.3.4.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new dc40659 HIVE-21585: Upgrade Hive branch-2.3 to ORC.1.3.4.
dc40659 is described below
commit dc40659c627528a174e00f018724360e1f066bab
Author: Owen O'Malley <om...@apache.org>
AuthorDate: Fri Apr 5 10:41:15 2019 -0700
HIVE-21585: Upgrade Hive branch-2.3 to ORC.1.3.4.
Fixes #590
Signed-off-by: Owen O'Malley <om...@apache.org>
---
.travis.yml | 4 +---
.../apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java | 6 ++++++
.../apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java | 5 +++++
.../hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java | 6 ++++++
pom.xml | 2 +-
.../org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java | 4 ++--
.../java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java | 2 +-
7 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index d0e1568..98057b6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,10 +23,8 @@ dist: trusty
# that requires full git history, enable this
# before_install: git fetch --unshallow
-# parallel builds on jdk7 and jdk8
language: java
jdk:
- - oraclejdk7
- oraclejdk8
cache:
@@ -44,4 +42,4 @@ before_install:
install: true
-script: mvn clean install -DskipTests -T 4 -q -Pitests
+script: travis_wait 30 mvn clean install -DskipTests -T 4 -q -Pitests
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 03bc3ce..07318da 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hive.llap.counters.LlapIOCounters;
+import org.apache.orc.CompressionCodec;
import org.apache.orc.TypeDescription;
import org.apache.orc.impl.DataReaderProperties;
import org.apache.orc.impl.OrcIndex;
@@ -884,6 +885,11 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
}
@Override
+ public CompressionCodec getCompressionCodec() {
+ return orcDataReader.getCompressionCodec();
+ }
+
+ @Override
public DiskRangeList readFileData(DiskRangeList range, long baseOffset,
boolean doForceDirect) throws IOException {
long startTime = counters.startTimeCounter();
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
index 907200a..dd39fd7 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
@@ -546,6 +546,11 @@ public class SerDeEncodedDataReader extends CallableWithNdc<Void>
throw new UnsupportedOperationException(); // Only used in ACID writer.
}
+ @Override
+ public CompressionCodec getCompressionCodec() {
+ return null;
+ }
+
public void setCurrentStripeOffsets(long currentKnownTornStart,
long firstStartOffset, long lastStartOffset, long currentFileOffset) {
currentStripe.knownTornStart = currentKnownTornStart;
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
index 13c7767..2c41d21 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestIncrementalObjectSizeEstimator.java
@@ -28,6 +28,7 @@ import java.util.ArrayList;
import java.util.LinkedHashSet;
import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.orc.CompressionCodec;
import org.apache.orc.DataReader;
import org.apache.orc.OrcFile;
import org.apache.orc.TypeDescription;
@@ -159,6 +160,11 @@ public class TestIncrementalObjectSizeEstimator {
@Override
public void close() throws IOException {
}
+
+ @Override
+ public CompressionCodec getCompressionCodec() {
+ return null;
+ }
}
@Test
diff --git a/pom.xml b/pom.xml
index 736eef7..adc2541 100644
--- a/pom.xml
+++ b/pom.xml
@@ -175,7 +175,7 @@
<libthrift.version>0.9.3</libthrift.version>
<log4j2.version>2.6.2</log4j2.version>
<opencsv.version>2.3</opencsv.version>
- <orc.version>1.3.3</orc.version>
+ <orc.version>1.3.4</orc.version>
<mockito-all.version>1.9.5</mockito-all.version>
<mina.version>2.0.0-M5</mina.version>
<netty.version>4.0.52.Final</netty.version>
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index 1252e40..420bc29 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -122,11 +122,11 @@ class EncodedReaderImpl implements EncodedReader {
private final DataCache cacheWrapper;
private boolean isTracingEnabled;
- public EncodedReaderImpl(Object fileKey, List<OrcProto.Type> types, CompressionCodec codec,
+ public EncodedReaderImpl(Object fileKey, List<OrcProto.Type> types,
int bufferSize, long strideRate, DataCache cacheWrapper, DataReader dataReader,
PoolFactory pf) throws IOException {
this.fileKey = fileKey;
- this.codec = codec;
+ this.codec = dataReader.getCompressionCodec();
this.types = types;
this.bufferSize = bufferSize;
this.rowIndexStride = strideRate;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java
index 4856fb3..25645a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java
@@ -36,6 +36,6 @@ class ReaderImpl extends org.apache.hadoop.hive.ql.io.orc.ReaderImpl implements
public EncodedReader encodedReader(
Object fileKey, DataCache dataCache, DataReader dataReader, PoolFactory pf) throws IOException {
return new EncodedReaderImpl(fileKey, types,
- codec, bufferSize, rowIndexStride, dataCache, dataReader, pf);
+ bufferSize, rowIndexStride, dataCache, dataReader, pf);
}
}