You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by fo...@apache.org on 2021/04/08 20:36:30 UTC
[parquet-mr] branch master updated: PARQUET-2005: Upgrade thrift to
0.14.1 (#884)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new d23a9a7 PARQUET-2005: Upgrade thrift to 0.14.1 (#884)
d23a9a7 is described below
commit d23a9a79927a03ce6b8e192436c64ad34c36d67b
Author: Gabor Szadovszky <ga...@apache.org>
AuthorDate: Thu Apr 8 22:35:49 2021 +0200
PARQUET-2005: Upgrade thrift to 0.14.1 (#884)
* PARQUET-2005: Upgrade thrift to 0.14.1
* PARQUET-2005: Update thrift version in CI scripts and README
* Update README.md
Co-authored-by: Fokko Driesprong <fo...@driesprong.frl>
Co-authored-by: Fokko Driesprong <fo...@driesprong.frl>
---
README.md | 12 ++++++------
dev/ci-before_install.sh | 2 +-
.../src/main/java/org/apache/parquet/format/CliUtils.java | 2 +-
.../java/org/apache/parquet/format/InterningProtocol.java | 5 +++++
.../src/main/java/org/apache/parquet/format/Util.java | 5 +++--
.../parquet/hadoop/thrift/ThriftBytesWriteSupport.java | 3 ++-
.../apache/parquet/thrift/BufferedProtocolReadToWrite.java | 5 +++++
.../java/org/apache/parquet/thrift/ParquetProtocol.java | 5 +++++
.../org/apache/parquet/thrift/TestProtocolReadToWrite.java | 5 +++--
pom.xml | 13 +++----------
10 files changed, 34 insertions(+), 23 deletions(-)
diff --git a/README.md b/README.md
index 7b10746..5ad4570 100644
--- a/README.md
+++ b/README.md
@@ -35,19 +35,19 @@ Parquet-MR uses Maven to build and depends on the thrift compiler (protoc is now
To build and install the thrift compiler, run:
```
-wget -nv http://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.tar.gz
-tar xzf thrift-0.13.0.tar.gz
-cd thrift-0.13.0
+wget -nv http://archive.apache.org/dist/thrift/0.14.1/thrift-0.14.1.tar.gz
+tar xzf thrift-0.14.1.tar.gz
+cd thrift-0.14.1
chmod +x ./configure
./configure --disable-libs
sudo make install
```
-If you're on OSX and use homebrew, you can instead install Thrift 0.13.0 with `brew` and ensure that it comes first in your `PATH`.
+If you're on OSX and use homebrew, you can instead install Thrift 0.14.1 with `brew` and ensure that it comes first in your `PATH`.
```
-brew install thrift@0.13
-export PATH="/usr/local/opt/thrift@0.13.0/bin:$PATH"
+brew install thrift
+export PATH="/usr/local/opt/thrift@0.14.1/bin:$PATH"
```
### Build Parquet with Maven
diff --git a/dev/ci-before_install.sh b/dev/ci-before_install.sh
index d18b367..47b71d5 100755
--- a/dev/ci-before_install.sh
+++ b/dev/ci-before_install.sh
@@ -20,7 +20,7 @@
# This script gets invoked by the CI system in a "before install" step
################################################################################
-export THRIFT_VERSION=0.13.0
+export THRIFT_VERSION=0.14.1
set -e
date
diff --git a/parquet-format-structures/src/main/java/org/apache/parquet/format/CliUtils.java b/parquet-format-structures/src/main/java/org/apache/parquet/format/CliUtils.java
index 3739b64..b8800c8 100644
--- a/parquet-format-structures/src/main/java/org/apache/parquet/format/CliUtils.java
+++ b/parquet-format-structures/src/main/java/org/apache/parquet/format/CliUtils.java
@@ -40,8 +40,8 @@ public class CliUtils {
* @throws IOException if any Thrift error occurs during the serialization
*/
public static String toJson(TBase<?, ?> tbase) throws IOException {
- TSerializer serializer = new TSerializer(new TSimpleJSONProtocol.Factory());
try {
+ TSerializer serializer = new TSerializer(new TSimpleJSONProtocol.Factory());
return serializer.toString(tbase);
} catch (TException e) {
// Wrapping the exception the not to expose the shaded Thrift class TException
diff --git a/parquet-format-structures/src/main/java/org/apache/parquet/format/InterningProtocol.java b/parquet-format-structures/src/main/java/org/apache/parquet/format/InterningProtocol.java
index a405d4f..a1a6466 100644
--- a/parquet-format-structures/src/main/java/org/apache/parquet/format/InterningProtocol.java
+++ b/parquet-format-structures/src/main/java/org/apache/parquet/format/InterningProtocol.java
@@ -228,4 +228,9 @@ public class InterningProtocol extends TProtocol {
return delegate.toString();
}
+ @Override
+ public int getMinSerializedSize(byte type) throws TException {
+ return delegate.getMinSerializedSize(type);
+ }
+
}
diff --git a/parquet-format-structures/src/main/java/org/apache/parquet/format/Util.java b/parquet-format-structures/src/main/java/org/apache/parquet/format/Util.java
index 36aaf74..32c1986 100644
--- a/parquet-format-structures/src/main/java/org/apache/parquet/format/Util.java
+++ b/parquet-format-structures/src/main/java/org/apache/parquet/format/Util.java
@@ -45,6 +45,7 @@ import org.apache.thrift.protocol.TCompactProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TIOStreamTransport;
import org.apache.thrift.transport.TMemoryBuffer;
+import org.apache.thrift.transport.TTransportException;
import org.apache.parquet.format.event.Consumers.Consumer;
import org.apache.parquet.format.event.Consumers.DelegatingFieldConsumer;
import org.apache.parquet.format.event.EventBasedThriftReader;
@@ -336,11 +337,11 @@ public class Util {
}
}
- private static TProtocol protocol(OutputStream to) {
+ private static TProtocol protocol(OutputStream to) throws TTransportException {
return protocol(new TIOStreamTransport(to));
}
- private static TProtocol protocol(InputStream from) {
+ private static TProtocol protocol(InputStream from) throws TTransportException {
return protocol(new TIOStreamTransport(from));
}
diff --git a/parquet-thrift/src/main/java/org/apache/parquet/hadoop/thrift/ThriftBytesWriteSupport.java b/parquet-thrift/src/main/java/org/apache/parquet/hadoop/thrift/ThriftBytesWriteSupport.java
index bdb6578..6b9d75d 100644
--- a/parquet-thrift/src/main/java/org/apache/parquet/hadoop/thrift/ThriftBytesWriteSupport.java
+++ b/parquet-thrift/src/main/java/org/apache/parquet/hadoop/thrift/ThriftBytesWriteSupport.java
@@ -30,6 +30,7 @@ import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.transport.TIOStreamTransport;
+import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.parquet.hadoop.BadConfigurationException;
@@ -156,7 +157,7 @@ public class ThriftBytesWriteSupport extends WriteSupport<BytesWritable> {
}
}
- private TProtocol protocol(BytesWritable record) {
+ private TProtocol protocol(BytesWritable record) throws TTransportException {
TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(new ByteArrayInputStream(record.getBytes())));
/* Reduce the chance of OOM when data is corrupted. When readBinary is called on TBinaryProtocol, it reads the length of the binary first,
diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/BufferedProtocolReadToWrite.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/BufferedProtocolReadToWrite.java
index 2e76642..254386b 100644
--- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/BufferedProtocolReadToWrite.java
+++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/BufferedProtocolReadToWrite.java
@@ -728,6 +728,11 @@ public class BufferedProtocolReadToWrite implements ProtocolPipe {
public ByteBuffer readBinary() throws TException {
return null;
}
+
+ @Override
+ public int getMinSerializedSize(byte type) throws TException {
+ return 0;
+ }
}
}
diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/ParquetProtocol.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/ParquetProtocol.java
index 3eb431e..b26cdcb 100644
--- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/ParquetProtocol.java
+++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/ParquetProtocol.java
@@ -274,4 +274,9 @@ public abstract class ParquetProtocol extends TProtocol {
throw exception();
}
+ @Override
+ public int getMinSerializedSize(byte type) throws TException {
+ throw exception();
+ }
+
}
diff --git a/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestProtocolReadToWrite.java b/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestProtocolReadToWrite.java
index bdd3a3f..c1614e8 100644
--- a/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestProtocolReadToWrite.java
+++ b/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestProtocolReadToWrite.java
@@ -25,6 +25,7 @@ import org.apache.thrift.TException;
import org.apache.thrift.protocol.TCompactProtocol;
import org.apache.thrift.protocol.TField;
import org.apache.thrift.transport.TIOStreamTransport;
+import org.apache.thrift.transport.TTransportException;
import org.junit.Test;
import org.apache.parquet.thrift.test.Phone;
import org.apache.parquet.thrift.test.StructWithExtraField;
@@ -358,11 +359,11 @@ public class TestProtocolReadToWrite {
assertEquals(1, countingHandler.fieldIgnoredCount);
}
- private TCompactProtocol protocol(OutputStream to) {
+ private TCompactProtocol protocol(OutputStream to) throws TTransportException {
return new TCompactProtocol(new TIOStreamTransport(to));
}
- private TCompactProtocol protocol(InputStream from) {
+ private TCompactProtocol protocol(InputStream from) throws TTransportException {
return new TCompactProtocol(new TIOStreamTransport(from));
}
diff --git a/pom.xml b/pom.xml
index a005873..691bbb5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,7 +85,7 @@
<parquet.format.version>2.8.0</parquet.format.version>
<previous.version>1.12.0</previous.version>
<thrift.executable>thrift</thrift.executable>
- <format.thrift.executable>thrift</format.thrift.executable>
+ <format.thrift.executable>${thrift.executable}</format.thrift.executable>
<scala.version>2.12.8</scala.version>
<!-- scala.binary.version is used for projects that fetch dependencies that are in scala -->
<scala.binary.version>2.12</scala.binary.version>
@@ -94,8 +94,8 @@
<pig.version>0.16.0</pig.version>
<pig.classifier>h2</pig.classifier>
<thrift-maven-plugin.version>0.10.0</thrift-maven-plugin.version>
- <thrift.version>0.13.0</thrift.version>
- <format.thrift.version>0.13.0</format.thrift.version>
+ <thrift.version>0.14.1</thrift.version>
+ <format.thrift.version>${thrift.version}</format.thrift.version>
<fastutil.version>8.4.2</fastutil.version>
<semver.api.version>0.9.33</semver.api.version>
<slf4j.version>1.7.22</slf4j.version>
@@ -586,13 +586,6 @@
</build>
</profile>
- <profile>
- <id>thrift9</id>
- <properties>
- <thrift.version>0.9.0</thrift.version>
- </properties>
- </profile>
-
<!-- Profile for CI tests to have less output -->
<profile>
<id>ci-test</id>