You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2022/09/24 09:41:19 UTC
[pinot] branch master updated: Remove netty server query support from presto-pinot-driver to remove pinot-core and pinot-segment-local dependencies (#9455)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e2d8aaab72 Remove netty server query support from presto-pinot-driver to remove pinot-core and pinot-segment-local dependencies (#9455)
e2d8aaab72 is described below
commit e2d8aaab7236aa8dccf6b5cfaefd3d98ea0e4639
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Sat Sep 24 02:41:11 2022 -0700
Remove netty server query support from presto-pinot-driver to remove pinot-core and pinot-segment-local dependencies (#9455)
---
.../pinot-core-jdk8/pom.xml | 298 ------------------
.../pinot-core-jdk8/src | 1 -
.../pinot-segment-local-jdk8/pom.xml | 147 ---------
.../pinot-segment-local-jdk8/src | 1 -
.../prestodb-pinot-dependencies/pom.xml | 2 -
.../presto-pinot-driver/pom.xml | 80 -----
.../presto/PinotScatterGatherQueryClient.java | 341 ---------------------
7 files changed, 870 deletions(-)
diff --git a/pinot-connectors/prestodb-pinot-dependencies/pinot-core-jdk8/pom.xml b/pinot-connectors/prestodb-pinot-dependencies/pinot-core-jdk8/pom.xml
deleted file mode 100644
index a459e16327..0000000000
--- a/pinot-connectors/prestodb-pinot-dependencies/pinot-core-jdk8/pom.xml
+++ /dev/null
@@ -1,298 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>prestodb-pinot-dependencies</artifactId>
- <groupId>org.apache.pinot</groupId>
- <version>0.12.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pinot-core-jdk8</artifactId>
- <name>Pinot Core JDK 8</name>
- <url>https://pinot.apache.org/</url>
- <properties>
- <pinot.root>${basedir}/../../..</pinot.root>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>com.uber</groupId>
- <artifactId>h3</artifactId>
- </dependency>
- <dependency>
- <groupId>org.roaringbitmap</groupId>
- <artifactId>RoaringBitmap</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-spi-jdk8</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-common-jdk8</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-segment-spi-jdk8</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-segment-local-jdk8</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <!--<dependency>
- <groupId>org.apache.helix</groupId>
- <artifactId>helix-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>-->
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-native-epoll</artifactId>
- <classifier>linux-x86_64</classifier>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-native-kqueue</artifactId>
- <classifier>osx-x86_64</classifier>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-tcnative-boringssl-static</artifactId>
- <classifier>linux-x86_64</classifier>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-tcnative-boringssl-static</artifactId>
- <classifier>osx-x86_64</classifier>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math</artifactId>
- </dependency>
- <dependency>
- <groupId>com.clearspring.analytics</groupId>
- <artifactId>stream</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.datasketches</groupId>
- <artifactId>datasketches-java</artifactId>
- </dependency>
- <dependency>
- <groupId>com.tdunning</groupId>
- <artifactId>t-digest</artifactId>
- </dependency>
- <dependency>
- <groupId>org.xerial.larray</groupId>
- <artifactId>larray-mmap</artifactId>
- </dependency>
- <dependency>
- <groupId>net.sf.jopt-simple</groupId>
- <artifactId>jopt-simple</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>org.locationtech.jts</groupId>
- <artifactId>jts-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>jersey-container-grizzly2-http</artifactId>
- </dependency>
- <dependency>
- <groupId>org.glassfish.grizzly</groupId>
- <artifactId>grizzly-http-server</artifactId>
- </dependency>
- <dependency>
- <groupId>org.glassfish.hk2</groupId>
- <artifactId>hk2-locator</artifactId>
- </dependency>
-
- <!-- test -->
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-common-jdk8</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-segment-local-jdk8</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>nl.jqno.equalsverifier</groupId>
- <artifactId>equalsverifier</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-avro</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-csv</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-json</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-spi-jdk8</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-yammer</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>${lucene.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-queryparser</artifactId>
- <version>${lucene.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
- <version>${lucene.version}</version>
- </dependency>
- </dependencies>
- <profiles>
- <profile>
- <id>build-shaded-jar</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <relocations>
- <relocation>
- <pattern>com.google.common.base</pattern>
- <shadedPattern>${shade.prefix}.com.google.common.base</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.http</pattern>
- <shadedPattern>${shade.prefix}.org.apache.http</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
diff --git a/pinot-connectors/prestodb-pinot-dependencies/pinot-core-jdk8/src b/pinot-connectors/prestodb-pinot-dependencies/pinot-core-jdk8/src
deleted file mode 120000
index 020b801bb2..0000000000
--- a/pinot-connectors/prestodb-pinot-dependencies/pinot-core-jdk8/src
+++ /dev/null
@@ -1 +0,0 @@
-../../../pinot-core/src
\ No newline at end of file
diff --git a/pinot-connectors/prestodb-pinot-dependencies/pinot-segment-local-jdk8/pom.xml b/pinot-connectors/prestodb-pinot-dependencies/pinot-segment-local-jdk8/pom.xml
deleted file mode 100644
index 8f40798b20..0000000000
--- a/pinot-connectors/prestodb-pinot-dependencies/pinot-segment-local-jdk8/pom.xml
+++ /dev/null
@@ -1,147 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>prestodb-pinot-dependencies</artifactId>
- <groupId>org.apache.pinot</groupId>
- <version>0.12.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pinot-segment-local-jdk8</artifactId>
- <name>Pinot local segment implementations JDK 8</name>
- <url>https://pinot.apache.org/</url>
- <properties>
- <pinot.root>${basedir}/../../..</pinot.root>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-spi-jdk8</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-common-jdk8</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-segment-spi-jdk8</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.xerial.larray</groupId>
- <artifactId>larray-mmap</artifactId>
- </dependency>
- <dependency>
- <groupId>com.uber</groupId>
- <artifactId>h3</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>${lucene.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-queryparser</artifactId>
- <version>${lucene.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
- <version>${lucene.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.datasketches</groupId>
- <artifactId>datasketches-java</artifactId>
- </dependency>
- <dependency>
- <groupId>com.tdunning</groupId>
- <artifactId>t-digest</artifactId>
- </dependency>
- <dependency>
- <groupId>com.clearspring.analytics</groupId>
- <artifactId>stream</artifactId>
- </dependency>
- <dependency>
- <groupId>org.quartz-scheduler</groupId>
- <artifactId>quartz</artifactId>
- </dependency>
-
- <!-- Test dependencies -->
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- <scope>test</scope>
- </dependency>
- <!-- required for static mock in IndexCreatorOverrideTest -->
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-inline</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-avro</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-csv</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-json</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-spi-jdk8</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-yammer</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-common-jdk8</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
-</project>
diff --git a/pinot-connectors/prestodb-pinot-dependencies/pinot-segment-local-jdk8/src b/pinot-connectors/prestodb-pinot-dependencies/pinot-segment-local-jdk8/src
deleted file mode 120000
index 538a29d490..0000000000
--- a/pinot-connectors/prestodb-pinot-dependencies/pinot-segment-local-jdk8/src
+++ /dev/null
@@ -1 +0,0 @@
-../../../pinot-segment-local/src
\ No newline at end of file
diff --git a/pinot-connectors/prestodb-pinot-dependencies/pom.xml b/pinot-connectors/prestodb-pinot-dependencies/pom.xml
index 53edd2fb2d..a496ca63bb 100644
--- a/pinot-connectors/prestodb-pinot-dependencies/pom.xml
+++ b/pinot-connectors/prestodb-pinot-dependencies/pom.xml
@@ -43,10 +43,8 @@
<modules>
<module>pinot-spi-jdk8</module>
<module>pinot-common-jdk8</module>
- <module>pinot-core-jdk8</module>
<module>pinot-java-client-jdk8</module>
<module>pinot-segment-spi-jdk8</module>
- <module>pinot-segment-local-jdk8</module>
<module>presto-pinot-driver</module>
</modules>
diff --git a/pinot-connectors/prestodb-pinot-dependencies/presto-pinot-driver/pom.xml b/pinot-connectors/prestodb-pinot-dependencies/presto-pinot-driver/pom.xml
index 892c235cc5..e99bab8145 100644
--- a/pinot-connectors/prestodb-pinot-dependencies/presto-pinot-driver/pom.xml
+++ b/pinot-connectors/prestodb-pinot-dependencies/presto-pinot-driver/pom.xml
@@ -242,86 +242,6 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-core-jdk8</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.antlr</groupId>
- <artifactId>antlr4-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.osgi</groupId>
- <artifactId>org.osgi.core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>org.apache.helix</groupId>
<artifactId>helix-core</artifactId>
diff --git a/pinot-connectors/prestodb-pinot-dependencies/presto-pinot-driver/src/main/java/org/apache/pinot/connector/presto/PinotScatterGatherQueryClient.java b/pinot-connectors/prestodb-pinot-dependencies/presto-pinot-driver/src/main/java/org/apache/pinot/connector/presto/PinotScatterGatherQueryClient.java
deleted file mode 100644
index 28ceff307e..0000000000
--- a/pinot-connectors/prestodb-pinot-dependencies/presto-pinot-driver/src/main/java/org/apache/pinot/connector/presto/PinotScatterGatherQueryClient.java
+++ /dev/null
@@ -1,341 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.connector.presto;
-
-import com.google.common.collect.ImmutableMap;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
-import org.apache.helix.model.InstanceConfig;
-import org.apache.pinot.common.config.TlsConfig;
-import org.apache.pinot.common.metrics.BrokerMetrics;
-import org.apache.pinot.common.request.BrokerRequest;
-import org.apache.pinot.common.utils.DataTable;
-import org.apache.pinot.connector.presto.plugin.metrics.NoopPinotMetricFactory;
-import org.apache.pinot.core.transport.AsyncQueryResponse;
-import org.apache.pinot.core.transport.QueryRouter;
-import org.apache.pinot.core.transport.ServerInstance;
-import org.apache.pinot.core.transport.ServerResponse;
-import org.apache.pinot.core.transport.ServerRoutingInstance;
-import org.apache.pinot.core.transport.server.routing.stats.ServerRoutingStatsManager;
-import org.apache.pinot.spi.config.table.TableType;
-import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.metrics.PinotMetricUtils;
-import org.apache.pinot.spi.utils.builder.TableNameBuilder;
-import org.apache.pinot.sql.parsers.CalciteSqlCompiler;
-
-import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_METRICS_FACTORY_CLASS_NAME;
-
-
-public class PinotScatterGatherQueryClient {
- private static final String PRESTO_HOST_PREFIX = "presto-pinot-";
-
- private final String _prestoHostId;
- private final BrokerMetrics _brokerMetrics;
- private final Queue<QueryRouter> _queryRouters = new ConcurrentLinkedQueue<>();
- private final Config _config;
- private final Map<String, AtomicInteger> _concurrentQueriesCountMap = new ConcurrentHashMap<>();
-
- public enum ErrorCode {
- PINOT_INSUFFICIENT_SERVER_RESPONSE(true),
- PINOT_INVALID_SQL_GENERATED(false),
- PINOT_UNCLASSIFIED_ERROR(false),
- PINOT_QUERY_BACKLOG_FULL(false);
-
- private final boolean _retriable;
-
- ErrorCode(boolean retriable) {
- _retriable = retriable;
- }
-
- public boolean isRetriable() {
- return _retriable;
- }
- }
-
- public static class PinotException extends RuntimeException {
- private final ErrorCode _errorCode;
-
- public PinotException(ErrorCode errorCode, String message, Throwable t) {
- super(message, t);
- _errorCode = errorCode;
- }
-
- public PinotException(ErrorCode errorCode, String message) {
- this(errorCode, message, null);
- }
-
- public ErrorCode getErrorCode() {
- return _errorCode;
- }
- }
-
- public static class Config {
- private final int _threadPoolSize;
-
- private final int _maxBacklogPerServer;
-
- private TlsConfig _tlsConfig = new TlsConfig();
-
- @Deprecated
- private final long _idleTimeoutMillis;
- @Deprecated
- private final int _minConnectionsPerServer;
- @Deprecated
- private final int _maxConnectionsPerServer;
-
- public Config(Map<String, Object> pinotConfigs) {
- _idleTimeoutMillis = Long.parseLong(pinotConfigs.get("idleTimeoutMillis").toString());
- _threadPoolSize = Integer.parseInt(pinotConfigs.get("threadPoolSize").toString());
- _minConnectionsPerServer = Integer.parseInt(pinotConfigs.get("minConnectionsPerServer").toString());
- _maxBacklogPerServer = Integer.parseInt(pinotConfigs.get("maxBacklogPerServer").toString());
- _maxConnectionsPerServer = Integer.parseInt(pinotConfigs.get("maxConnectionsPerServer").toString());
- _tlsConfig.setClientAuthEnabled(Boolean.parseBoolean(pinotConfigs.get("isClientAuthEnabled").toString()));
- _tlsConfig.setTrustStorePath(pinotConfigs.get("trustStorePath").toString());
- _tlsConfig.setTrustStorePassword(pinotConfigs.get("trustStorePassword").toString());
- _tlsConfig.setTrustStoreType(pinotConfigs.get("trustStoreType").toString());
- _tlsConfig.setKeyStorePath(pinotConfigs.get("keyStorePath").toString());
- _tlsConfig.setKeyStorePassword(pinotConfigs.get("keyStorePassword").toString());
- _tlsConfig.setKeyStoreType(pinotConfigs.get("keyStoreType").toString());
- _tlsConfig.setSslProvider(pinotConfigs.get("sslProvider").toString());
- }
-
- public Config(long idleTimeoutMillis, int threadPoolSize, int minConnectionsPerServer, int maxBacklogPerServer,
- int maxConnectionsPerServer) {
- _idleTimeoutMillis = idleTimeoutMillis;
- _threadPoolSize = threadPoolSize;
- _minConnectionsPerServer = minConnectionsPerServer;
- _maxBacklogPerServer = maxBacklogPerServer;
- _maxConnectionsPerServer = maxConnectionsPerServer;
- _tlsConfig.setClientAuthEnabled(false);
- }
-
- public int getThreadPoolSize() {
- return _threadPoolSize;
- }
-
- public int getMaxBacklogPerServer() {
- return _maxBacklogPerServer;
- }
-
- @Deprecated
- public long getIdleTimeoutMillis() {
- return _idleTimeoutMillis;
- }
-
- @Deprecated
- public int getMinConnectionsPerServer() {
- return _minConnectionsPerServer;
- }
-
- @Deprecated
- public int getMaxConnectionsPerServer() {
- return _maxConnectionsPerServer;
- }
-
- public boolean isClientAuthEnabled() {
- return _tlsConfig.isClientAuthEnabled();
- }
-
- public String getTrustStoreType() {
- return _tlsConfig.getTrustStoreType();
- }
-
- public String getTrustStorePath() {
- return _tlsConfig.getTrustStorePath();
- }
-
- public String getTrustStorePassword() {
- return _tlsConfig.getTrustStorePassword();
- }
-
- public String getKeyStoreType() {
- return _tlsConfig.getKeyStoreType();
- }
-
- public String getKeyStorePath() {
- return _tlsConfig.getKeyStorePath();
- }
-
- public String getKeyStorePassword() {
- return _tlsConfig.getKeyStorePassword();
- }
-
- public String getSslProvider() {
- return _tlsConfig.getSslProvider();
- }
- }
-
- public PinotScatterGatherQueryClient(Config pinotConfig) {
- _prestoHostId = getDefaultPrestoId();
- PinotMetricUtils.init(new PinotConfiguration(
- ImmutableMap.of(CONFIG_OF_METRICS_FACTORY_CLASS_NAME, NoopPinotMetricFactory.class.getName())));
- _brokerMetrics = new BrokerMetrics(PinotMetricUtils.getPinotMetricsRegistry());
- _brokerMetrics.initializeGlobalMeters();
- TlsConfig tlsConfig = getTlsConfig(pinotConfig);
- ServerRoutingStatsManager serverRoutingStatsManager = new ServerRoutingStatsManager(new PinotConfiguration());
-
- // Setup QueryRouters
- for (int i = 0; i < pinotConfig.getThreadPoolSize(); i++) {
- _queryRouters.add(new QueryRouter(String.format("%s-%d", _prestoHostId, i), _brokerMetrics, null, tlsConfig,
- serverRoutingStatsManager));
- }
-
- _config = pinotConfig;
- }
-
- private TlsConfig getTlsConfig(Config pinotConfig) {
- TlsConfig tlsConfig = new TlsConfig();
- tlsConfig.setClientAuthEnabled(pinotConfig.isClientAuthEnabled());
- tlsConfig.setTrustStoreType(pinotConfig.getTrustStoreType());
- tlsConfig.setTrustStorePath(pinotConfig.getTrustStorePath());
- tlsConfig.setTrustStorePassword(pinotConfig.getTrustStorePassword());
- tlsConfig.setKeyStoreType(pinotConfig.getKeyStoreType());
- tlsConfig.setKeyStorePath(pinotConfig.getKeyStorePath());
- tlsConfig.setKeyStorePassword(pinotConfig.getKeyStorePassword());
- tlsConfig.setSslProvider(pinotConfig.getSslProvider());
- return tlsConfig;
- }
-
- private static <T> T doWithRetries(int retries, Function<Integer, T> caller) {
- PinotException firstError = null;
- for (int i = 0; i < retries; i++) {
- try {
- return caller.apply(i);
- } catch (PinotException e) {
- if (firstError == null) {
- firstError = e;
- }
- if (!e.getErrorCode().isRetriable()) {
- throw e;
- }
- }
- }
- throw firstError;
- }
-
- private String getDefaultPrestoId() {
- String defaultBrokerId;
- try {
- defaultBrokerId = PRESTO_HOST_PREFIX + InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- defaultBrokerId = PRESTO_HOST_PREFIX;
- }
- return defaultBrokerId;
- }
-
- public Map<ServerInstance, DataTable> queryPinotServerForDataTable(String query, String serverHost,
- List<String> segments, long connectionTimeoutInMillis, boolean ignoreEmptyResponses, int pinotRetryCount) {
- BrokerRequest brokerRequest;
- try {
- brokerRequest = CalciteSqlCompiler.compileToBrokerRequest(query);
- } catch (Exception e) {
- throw new PinotException(ErrorCode.PINOT_INVALID_SQL_GENERATED,
- String.format("Parsing error with on %s, Error = %s", serverHost, e.getMessage()), e);
- }
-
- Map<ServerInstance, List<String>> routingTable = new HashMap<>();
- routingTable.put(new ServerInstance(new InstanceConfig(serverHost)), new ArrayList<>(segments));
-
- // Unfortunately the retries will all hit the same server because the routing decision has already been made by
- // the pinot broker
- Map<ServerInstance, DataTable> serverResponseMap = doWithRetries(pinotRetryCount, (requestId) -> {
- String rawTableName = TableNameBuilder.extractRawTableName(brokerRequest.getQuerySource().getTableName());
- if (!_concurrentQueriesCountMap.containsKey(serverHost)) {
- _concurrentQueriesCountMap.put(serverHost, new AtomicInteger(0));
- }
- int concurrentQueryNum = _concurrentQueriesCountMap.get(serverHost).get();
- if (concurrentQueryNum > _config.getMaxBacklogPerServer()) {
- throw new PinotException(ErrorCode.PINOT_QUERY_BACKLOG_FULL,
- "Reaching server query max backlog size is - " + _config.getMaxBacklogPerServer());
- }
- _concurrentQueriesCountMap.get(serverHost).incrementAndGet();
- AsyncQueryResponse asyncQueryResponse;
- QueryRouter nextAvailableQueryRouter = getNextAvailableQueryRouter();
- if (TableNameBuilder.getTableTypeFromTableName(brokerRequest.getQuerySource().getTableName())
- == TableType.REALTIME) {
- asyncQueryResponse =
- nextAvailableQueryRouter.submitQuery(requestId, rawTableName, null, null, brokerRequest, routingTable,
- connectionTimeoutInMillis);
- } else {
- asyncQueryResponse =
- nextAvailableQueryRouter.submitQuery(requestId, rawTableName, brokerRequest, routingTable, null, null,
- connectionTimeoutInMillis);
- }
- Map<ServerInstance, DataTable> serverInstanceDataTableMap =
- gatherServerResponses(ignoreEmptyResponses, routingTable, asyncQueryResponse,
- brokerRequest.getQuerySource().getTableName());
- _queryRouters.offer(nextAvailableQueryRouter);
- _concurrentQueriesCountMap.get(serverHost).decrementAndGet();
- return serverInstanceDataTableMap;
- });
- return serverResponseMap;
- }
-
- private QueryRouter getNextAvailableQueryRouter() {
- QueryRouter queryRouter = _queryRouters.poll();
- while (queryRouter == null) {
- try {
- Thread.sleep(200L);
- } catch (InterruptedException e) {
- // Swallow the exception
- }
- queryRouter = _queryRouters.poll();
- }
- return queryRouter;
- }
-
- private Map<ServerInstance, DataTable> gatherServerResponses(boolean ignoreEmptyResponses,
- Map<ServerInstance, List<String>> routingTable, AsyncQueryResponse asyncQueryResponse, String tableNameWithType) {
- try {
- Map<ServerRoutingInstance, ServerResponse> queryResponses = asyncQueryResponse.getFinalResponses();
- if (!ignoreEmptyResponses) {
- if (queryResponses.size() != routingTable.size()) {
- Map<String, String> routingTableForLogging = new HashMap<>();
- routingTable.entrySet().forEach(entry -> {
- String valueToPrint = entry.getValue().size() > 10 ? String.format("%d segments", entry.getValue().size())
- : entry.getValue().toString();
- routingTableForLogging.put(entry.getKey().toString(), valueToPrint);
- });
- throw new PinotException(ErrorCode.PINOT_INSUFFICIENT_SERVER_RESPONSE,
- String.format("%d of %d servers responded with routing table servers: %s, query stats: %s",
- queryResponses.size(), routingTable.size(), routingTableForLogging,
- asyncQueryResponse.getServerStats()));
- }
- }
- Map<ServerInstance, DataTable> serverResponseMap = new HashMap<>();
- queryResponses.entrySet().forEach(entry -> serverResponseMap.put(new ServerInstance(
- new InstanceConfig(String.format("Server_%s_%d", entry.getKey().getHostname(),
- entry.getKey().getPort()))),
- entry.getValue().getDataTable()));
- return serverResponseMap;
- } catch (InterruptedException e) {
- throw new PinotException(ErrorCode.PINOT_UNCLASSIFIED_ERROR,
- String.format("Caught exception while fetching responses for table: %s", tableNameWithType), e);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org