You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2022/02/23 20:37:03 UTC
[pinot] branch master updated: [issue-8142] upgrade apache libs due to cves (#8143)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a68f61a [issue-8142] upgrade apache libs due to cves (#8143)
a68f61a is described below
commit a68f61a2f5a58d0d47793d9dc1d2372adc2e42ad
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Wed Feb 23 21:36:45 2022 +0100
[issue-8142] upgrade apache libs due to cves (#8143)
* [issue-8142] upgrade apache libs due to cves
catch exception
format issue
httpclient 4.5.13
Update ThriftRecordReader.java
Update ThriftRecordExtractorTest.java
* fix compile issue
* fix compile issue
* Update ThriftRecordExtractorTest.java
* Update ThriftRecordExtractorTest.java
* compile problem after merge
* try again to fix compile problem
---
LICENSE-binary | 15 ++++++++-------
.../src/main/java/org/apache/pinot/serde/SerDe.java | 10 ++++++++--
.../pinot/core/transport/InstanceRequestHandler.java | 8 +++++++-
.../org/apache/pinot/core/transport/ServerChannels.java | 11 +++++++++--
pinot-plugins/pinot-file-system/pinot-s3/pom.xml | 4 ++--
.../plugin/inputformat/thrift/ThriftRecordReader.java | 7 ++++++-
.../inputformat/thrift/ThriftRecordExtractorTest.java | 12 +++++-------
pom.xml | 16 ++++++++--------
8 files changed, 53 insertions(+), 30 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index dafe853..def59ed 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -256,10 +256,10 @@ com.yammer.metrics:metrics-core:2.2.0
com.zaxxer:HikariCP-java7:2.4.13
commons-cli:commons-cli:1.2
commons-codec:commons-codec:1.15
-commons-collections:commons-collections:3.2.1
+commons-collections:commons-collections:3.2.2
commons-configuration:commons-configuration:1.10
commons-httpclient:commons-httpclient:3.1
-commons-io:commons-io:2.4
+commons-io:commons-io:2.11.0
commons-lang:commons-lang:2.6
commons-logging:commons-logging:1.2
commons-pool:commons-pool:1.6
@@ -329,8 +329,9 @@ org.apache.calcite.avatica:avatica-core:1.13.0
org.apache.calcite:calcite-babel:1.29.0
org.apache.calcite:calcite-core:1.29.0
org.apache.calcite:calcite-linq4j:1.29.0
+org.apache.commons:commons-beanutils:1.9.4
org.apache.commons:commons-collections4:4.1
-org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-compress:1.21
org.apache.commons:commons-csv:1.0
org.apache.commons:commons-lang3:3.5
org.apache.commons:commons-math3:3.2
@@ -339,9 +340,9 @@ org.apache.datasketches:datasketches-java:1.2.0-incubating
org.apache.datasketches:datasketches-memory:1.2.0-incubating
org.apache.helix:helix-core:0.9.8
org.apache.hive:hive-storage-api:2.7.1
-org.apache.httpcomponents:httpclient:4.5.9
-org.apache.httpcomponents:httpcore:4.4.9
-org.apache.httpcomponents:httpmime:4.5.3
+org.apache.httpcomponents:httpclient:4.5.13
+org.apache.httpcomponents:httpcore:4.4.13
+org.apache.httpcomponents:httpmime:4.5.13
org.apache.kafka:kafka-clients:2.0.0
org.apache.kafka:kafka_2.10:0.9.0.1
org.apache.kafka:kafka_2.11:2.0.0
@@ -370,7 +371,7 @@ org.apache.pulsar:pulsar-client-api:2.7.2
org.apache.pulsar:pulsar-client-original:2.7.2
org.apache.pulsar:pulsar-common:2.7.2
org.apache.pulsar:pulsar-transaction-common:2.7.2
-org.apache.thrift:libthrift:0.12.0
+org.apache.thrift:libthrift:0.15.0
org.apache.yetus:audience-annotations:0.13.0
org.apache.zookeeper:zookeeper-jute:3.5.8
org.apache.zookeeper:zookeeper:3.5.8
diff --git a/pinot-common/src/main/java/org/apache/pinot/serde/SerDe.java b/pinot-common/src/main/java/org/apache/pinot/serde/SerDe.java
index 117786b..4d4890e 100644
--- a/pinot-common/src/main/java/org/apache/pinot/serde/SerDe.java
+++ b/pinot-common/src/main/java/org/apache/pinot/serde/SerDe.java
@@ -24,6 +24,7 @@ import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.TSerializer;
import org.apache.thrift.protocol.TProtocolFactory;
+import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,8 +48,13 @@ public class SerDe {
private final TDeserializer _deserializer;
public SerDe(TProtocolFactory factory) {
- _serializer = new TSerializer(factory);
- _deserializer = new TDeserializer(factory);
+ try {
+ _serializer = new TSerializer(factory);
+ _deserializer = new TDeserializer(factory);
+ } catch (TTransportException ttException) {
+ LOGGER.error("Unable to initialize Serde instance", ttException);
+ throw new RuntimeException("Unable to initialize Serde instance", ttException);
+ }
}
public byte[] serialize(@SuppressWarnings("rawtypes") TBase obj) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java
index 8ad6dcf..a931376 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/InstanceRequestHandler.java
@@ -45,6 +45,7 @@ import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,7 +60,7 @@ public class InstanceRequestHandler extends SimpleChannelInboundHandler<ByteBuf>
// TODO: make it configurable
private static final int SLOW_QUERY_LATENCY_THRESHOLD_MS = 100;
- private final TDeserializer _deserializer = new TDeserializer(new TCompactProtocol.Factory());
+ private final TDeserializer _deserializer;
private final QueryScheduler _queryScheduler;
private final ServerMetrics _serverMetrics;
private final AccessControl _accessControl;
@@ -69,6 +70,11 @@ public class InstanceRequestHandler extends SimpleChannelInboundHandler<ByteBuf>
_queryScheduler = queryScheduler;
_serverMetrics = serverMetrics;
_accessControl = accessControl;
+ try {
+ _deserializer = new TDeserializer(new TCompactProtocol.Factory());
+ } catch (TTransportException e) {
+ throw new RuntimeException("Failed to initialize Thrift Deserializer", e);
+ }
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerChannels.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerChannels.java
index 082653e..e2f783a 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerChannels.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/ServerChannels.java
@@ -45,6 +45,7 @@ import org.apache.pinot.common.request.InstanceRequest;
import org.apache.pinot.common.utils.TlsUtils;
import org.apache.thrift.TSerializer;
import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.transport.TTransportException;
/**
@@ -58,8 +59,7 @@ public class ServerChannels {
private final QueryRouter _queryRouter;
private final BrokerMetrics _brokerMetrics;
// TSerializer currently is not thread safe, must be put into a ThreadLocal.
- private final ThreadLocal<TSerializer> _threadLocalTSerializer =
- ThreadLocal.withInitial(() -> new TSerializer(new TCompactProtocol.Factory()));
+ private final ThreadLocal<TSerializer> _threadLocalTSerializer;
private final ConcurrentHashMap<ServerRoutingInstance, ServerChannel> _serverToChannelMap = new ConcurrentHashMap<>();
private final EventLoopGroup _eventLoopGroup = new NioEventLoopGroup();
private final TlsConfig _tlsConfig;
@@ -85,6 +85,13 @@ public class ServerChannels {
_queryRouter = queryRouter;
_brokerMetrics = brokerMetrics;
_tlsConfig = tlsConfig;
+ _threadLocalTSerializer = ThreadLocal.withInitial(() -> {
+ try {
+ return new TSerializer(new TCompactProtocol.Factory());
+ } catch (TTransportException e) {
+ throw new RuntimeException("Failed to initialize Thrift Serializer", e);
+ }
+ });
}
public void sendRequest(String rawTableName, AsyncQueryResponse asyncQueryResponse,
diff --git a/pinot-plugins/pinot-file-system/pinot-s3/pom.xml b/pinot-plugins/pinot-file-system/pinot-s3/pom.xml
index 506ab38..8d37e4e 100644
--- a/pinot-plugins/pinot-file-system/pinot-s3/pom.xml
+++ b/pinot-plugins/pinot-file-system/pinot-s3/pom.xml
@@ -37,8 +37,8 @@
<pinot.root>${basedir}/../../..</pinot.root>
<aws.sdk.version>2.14.28</aws.sdk.version>
<netty.version>4.1.54.Final</netty.version>
- <http.client.version>4.5.9</http.client.version>
- <http.core.version>4.4.9</http.core.version>
+ <http.client.version>4.5.13</http.client.version>
+ <http.core.version>4.4.13</http.core.version>
<s3mock.version>2.1.19</s3mock.version>
<javax.version>3.1.0</javax.version>
<phase.prop>package</phase.prop>
diff --git a/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordReader.java b/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordReader.java
index 2fcef1b..c7cba37 100644
--- a/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordReader.java
+++ b/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordReader.java
@@ -35,6 +35,7 @@ import org.apache.thrift.meta_data.FieldMetaData;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TIOStreamTransport;
+import org.apache.thrift.transport.TTransportException;
/**
@@ -80,7 +81,11 @@ public class ThriftRecordReader implements RecordReader {
private void init()
throws IOException {
_inputStream = RecordReaderUtils.getBufferedInputStream(_dataFile);
- _tProtocol = new TBinaryProtocol(new TIOStreamTransport(_inputStream));
+ try {
+ _tProtocol = new TBinaryProtocol(new TIOStreamTransport(_inputStream));
+ } catch (TTransportException e) {
+ throw new IOException(e);
+ }
_hasNext = hasMoreToRead();
}
diff --git a/pinot-plugins/pinot-input-format/pinot-thrift/src/test/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractorTest.java b/pinot-plugins/pinot-input-format/pinot-thrift/src/test/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractorTest.java
index 6fd3c63..78b69b2 100644
--- a/pinot-plugins/pinot-input-format/pinot-thrift/src/test/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractorTest.java
+++ b/pinot-plugins/pinot-input-format/pinot-thrift/src/test/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractorTest.java
@@ -129,16 +129,14 @@ public class ThriftRecordExtractorTest extends AbstractRecordExtractorTest {
thriftRecords.add(thriftRecord);
}
- BufferedOutputStream bufferedOut = new BufferedOutputStream(new FileOutputStream(_tempFile));
- TBinaryProtocol binaryOut = new TBinaryProtocol(new TIOStreamTransport(bufferedOut));
- for (ComplexTypes record : thriftRecords) {
- try {
+ try (BufferedOutputStream bufferedOut = new BufferedOutputStream(new FileOutputStream(_tempFile))) {
+ TBinaryProtocol binaryOut = new TBinaryProtocol(new TIOStreamTransport(bufferedOut));
+ for (ComplexTypes record : thriftRecords) {
record.write(binaryOut);
- } catch (TException e) {
- throw new IOException(e);
}
+ } catch (TException e) {
+ throw new IOException(e);
}
- bufferedOut.close();
}
private Map<String, Object> createRecord1() {
diff --git a/pom.xml b/pom.xml
index 0ebbfb9..2d637aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -299,17 +299,17 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
- <version>4.5.3</version>
+ <version>4.5.13</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.5.9</version>
+ <version>4.5.13</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.4.9</version>
+ <version>4.4.13</version>
</dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
@@ -471,7 +471,7 @@
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
- <version>3.2.1</version>
+ <version>3.2.2</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
@@ -486,7 +486,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.4</version>
+ <version>2.11.0</version>
</dependency>
<!-- zkclient & helix-core use netty -->
@@ -615,12 +615,12 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
- <version>1.20</version>
+ <version>1.21</version>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
- <version>0.12.0</version>
+ <version>0.15.0</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
@@ -756,7 +756,7 @@
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
- <version>1.8.3</version>
+ <version>1.9.4</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org