You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by so...@apache.org on 2018/09/15 14:54:53 UTC
[drill] branch master updated (b28355c -> 8f164bc)
This is an automated email from the ASF dual-hosted git repository.
sorabh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from b28355c DRILL-1248: Allow positional / named aliases in group by / having clauses
new 6ff854b DRILL-6625: Intermittent failures in Kafka unit tests Unit test changes to fix intermittent kafka producer and consumer errors.
new 8f164bc DRILL-3853: Upgrade to SqlLine 1.5.0 closes #1462
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
contrib/pom.xml | 1 -
contrib/sqlline/pom.xml | 54 ------
contrib/storage-kafka/pom.xml | 26 +++
.../exec/store/kafka/KafkaFilterPushdownTest.java | 5 +
.../exec/store/kafka/KafkaMessageGenerator.java | 5 +-
.../drill/exec/store/kafka/KafkaTestBase.java | 2 +-
.../drill/exec/store/kafka/TestKafkaSuit.java | 1 +
distribution/pom.xml | 4 -
distribution/src/assemble/bin.xml | 5 +
.../resources/drill-sqlline-override-example.conf | 27 ++-
distribution/src/resources/sqlline | 2 +-
distribution/src/resources/sqlline.bat | 17 +-
exec/java-exec/pom.xml | 4 +
.../drill/exec/client/DrillSqlLineApplication.java | 193 +++++++++++++++++++++
.../org/apache/drill/exec/server/Drillbit.java | 4 +
.../src/main/resources/drill-sqlline.conf | 88 ++++++++++
.../exec/client/DrillSqlLineApplicationTest.java | 113 ++++++++++++
.../resources/drill-sqlline-test-override.conf | 20 ++-
.../src/test/resources/drill-sqlline-test.conf | 36 ++--
exec/jdbc-all/pom.xml | 4 +
pom.xml | 7 +-
21 files changed, 502 insertions(+), 116 deletions(-)
delete mode 100644 contrib/sqlline/pom.xml
copy drill-yarn/src/test/resources/test-doy-config.conf => distribution/src/resources/drill-sqlline-override-example.conf (58%)
create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillSqlLineApplication.java
create mode 100644 exec/java-exec/src/main/resources/drill-sqlline.conf
create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/client/DrillSqlLineApplicationTest.java
copy distribution/src/resources/drill-conf => exec/java-exec/src/test/resources/drill-sqlline-test-override.conf (68%)
mode change 100755 => 100644
copy drill-yarn/src/test/resources/third-test-config.conf => exec/java-exec/src/test/resources/drill-sqlline-test.conf (58%)
[drill] 02/02: DRILL-3853: Upgrade to SqlLine 1.5.0 closes #1462
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sorabh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 8f164bc4081ea3b825553ec9215df5b068eec67a
Author: Arina Ielchiieva <ar...@gmail.com>
AuthorDate: Fri Sep 7 16:42:17 2018 +0000
DRILL-3853: Upgrade to SqlLine 1.5.0
closes #1462
---
contrib/pom.xml | 1 -
contrib/sqlline/pom.xml | 54 ------
distribution/pom.xml | 4 -
distribution/src/assemble/bin.xml | 5 +
.../resources/drill-sqlline-override-example.conf | 31 ++++
distribution/src/resources/sqlline | 2 +-
distribution/src/resources/sqlline.bat | 17 +-
exec/java-exec/pom.xml | 4 +
.../drill/exec/client/DrillSqlLineApplication.java | 193 +++++++++++++++++++++
.../org/apache/drill/exec/server/Drillbit.java | 4 +
.../src/main/resources/drill-sqlline.conf | 88 ++++++++++
.../exec/client/DrillSqlLineApplicationTest.java | 113 ++++++++++++
.../resources/drill-sqlline-test-override.conf | 30 ++++
.../src/test/resources/drill-sqlline-test.conf | 40 +++++
exec/jdbc-all/pom.xml | 4 +
pom.xml | 7 +-
16 files changed, 518 insertions(+), 79 deletions(-)
diff --git a/contrib/pom.xml b/contrib/pom.xml
index a96aa42..796e79b 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -43,7 +43,6 @@
<module>storage-kafka</module>
<module>storage-kudu</module>
<module>storage-opentsdb</module>
- <module>sqlline</module>
<module>data</module>
<module>udfs</module>
</modules>
diff --git a/contrib/sqlline/pom.xml b/contrib/sqlline/pom.xml
deleted file mode 100644
index 77070fc..0000000
--- a/contrib/sqlline/pom.xml
+++ /dev/null
@@ -1,54 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>drill-contrib-parent</artifactId>
- <groupId>org.apache.drill.contrib</groupId>
- <version>1.15.0-SNAPSHOT</version>
- </parent>
-
- <artifactId>drill-sqlline</artifactId>
- <name>contrib/sqlline</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.drill.exec</groupId>
- <artifactId>drill-jdbc</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>sqlline</groupId>
- <artifactId>sqlline</artifactId>
- </dependency>
- <!-- Specify xalan and xerces versions to avoid setXIncludeAware error. -->
- <dependency>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </dependency>
- <dependency>
- <groupId>xalan</groupId>
- <artifactId>xalan</artifactId>
- </dependency>
- </dependencies>
-</project>
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 35a274a..4952ca5 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -222,10 +222,6 @@
</activation>
<dependencies>
<dependency>
- <groupId>sqlline</groupId>
- <artifactId>sqlline</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.drill.contrib.data</groupId>
<artifactId>tpch-sample-data</artifactId>
<version>${project.version}</version>
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index c65d31a..85a4d5b 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -427,5 +427,10 @@
<outputDirectory>conf</outputDirectory>
<fileMode>0640</fileMode>
</file>
+ <file>
+ <source>src/resources/drill-sqlline-override-example.conf</source>
+ <outputDirectory>conf</outputDirectory>
+ <fileMode>0640</fileMode>
+ </file>
</files>
</assembly>
diff --git a/distribution/src/resources/drill-sqlline-override-example.conf b/distribution/src/resources/drill-sqlline-override-example.conf
new file mode 100644
index 0000000..34decdc
--- /dev/null
+++ b/distribution/src/resources/drill-sqlline-override-example.conf
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file customizes SqlLine application.
+# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+
+drill.sqlline {
+ # ${?drill.sqlline.quotes} is used to append new quotes to the existing ones
+ # remove it if you want to override existing quotes completely
+ quotes: ${?drill.sqlline.quotes} [
+ "east or west, drill is the best"
+ ],
+ # overrides default SqlLine properties
+ # for the full list of properties use !set command in SqlLine
+ opts: {
+ maxWidth: 10000,
+ timeout: -1
+ }
+}
diff --git a/distribution/src/resources/sqlline b/distribution/src/resources/sqlline
index 3f982bb..1db0083 100644
--- a/distribution/src/resources/sqlline
+++ b/distribution/src/resources/sqlline
@@ -82,7 +82,7 @@ if [[ ( ! $(ps -o stat= -p $$) =~ "+" ) && ! ( -p /dev/stdin ) ]]; then
fi
SHELL_OPTS="$DRILL_SHELL_JAVA_OPTS $SQLLINE_JAVA_OPTS $DRILL_SHELL_LOG_OPTS $CLIENT_GC_OPTS"
-CMD="$JAVA $SHELL_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver --maxWidth=10000"
+CMD="$JAVA $SHELL_OPTS -cp $CP sqlline.SqlLine -ac org.apache.drill.exec.client.DrillSqlLineApplication -d org.apache.drill.jdbc.Driver"
# The wrapper is purely for unit testing.
diff --git a/distribution/src/resources/sqlline.bat b/distribution/src/resources/sqlline.bat
index ac5a626..7821efa 100755
--- a/distribution/src/resources/sqlline.bat
+++ b/distribution/src/resources/sqlline.bat
@@ -200,28 +200,19 @@ set DRILL_CP=%DRILL_CP%;%DRILL_HOME%\jars\3rdparty\*
set DRILL_CP=%DRILL_CP%;%DRILL_HOME%\jars\classb\*
if NOT "test%DRILL_CLASSPATH%"=="test" set DRILL_CP=!DRILL_CP!;%DRILL_CLASSPATH%
-rem Override SQLLine's default initial transaction isolation level. (SQLLine
-rem sets an initial level instead of leaving it at whatever the Driver's default
-rem is.)
-rem Put our property specification before previous value of DRILL_SHELL_JAVA_OPTS
-rem so that it can still be overridden via DRILL_SHELL_JAVA_OPTS.
-rem
-rem This is not currently needed as the new SQLLine we are using doesn't isolate.
-rem set DRILL_SHELL_JAVA_OPTS=-Dsqlline.isolation=TRANSACTION_NONE %DRILL_SHELL_JAVA_OPTS%
-
set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% -Dlog.path="%DRILL_LOG_DIR%\sqlline.log" -Dlog.query.path="%DRILL_LOG_DIR%\sqlline_queries.log"
SET JAVA_CMD=%JAVA_HOME%\bin\%JAVA_EXE%
if "%JAVA_HOME%" == "" (set JAVA_CMD=%JAVA_EXE%)
set ERROR_CODE=0
-
+set SQLLINE_CALL=sqlline.SqlLine -ac org.apache.drill.exec.client.DrillSqlLineApplication -d org.apache.drill.jdbc.Driver
if NOT "test%QUERY%"=="test" (
- echo %QUERY% | "%JAVA_CMD%" %DRILL_SHELL_JAVA_OPTS% %DRILL_JAVA_OPTS% -cp "%DRILL_CP%" sqlline.SqlLine -d org.apache.drill.jdbc.Driver %DRILL_ARGS%
+ echo %QUERY% | "%JAVA_CMD%" %DRILL_SHELL_JAVA_OPTS% %DRILL_JAVA_OPTS% -cp "%DRILL_CP%" %SQLLINE_CALL% %DRILL_ARGS%
) else (
if NOT "test%FILE%"=="test" (
- "%JAVA_CMD%" %DRILL_SHELL_JAVA_OPTS% %DRILL_JAVA_OPTS% -cp "%DRILL_CP%" sqlline.SqlLine -d org.apache.drill.jdbc.Driver %DRILL_ARGS% --run=%FILE%
+ "%JAVA_CMD%" %DRILL_SHELL_JAVA_OPTS% %DRILL_JAVA_OPTS% -cp "%DRILL_CP%" %SQLLINE_CALL% %DRILL_ARGS% --run=%FILE%
) else (
- "%JAVA_CMD%" %DRILL_SHELL_JAVA_OPTS% %DRILL_JAVA_OPTS% -cp "%DRILL_CP%" sqlline.SqlLine -d org.apache.drill.jdbc.Driver %DRILL_ARGS%
+ "%JAVA_CMD%" %DRILL_SHELL_JAVA_OPTS% %DRILL_JAVA_OPTS% -cp "%DRILL_CP%" %SQLLINE_CALL% %DRILL_ARGS%
)
)
if ERRORLEVEL 1 goto error
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index e5b055b..5b00a8c 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -539,6 +539,10 @@
<artifactId>pcapngdecoder</artifactId>
<version>1.2</version>
</dependency>
+ <dependency>
+ <groupId>sqlline</groupId>
+ <artifactId>sqlline</artifactId>
+ </dependency>
</dependencies>
<profiles>
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillSqlLineApplication.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillSqlLineApplication.java
new file mode 100644
index 0000000..13d2b33
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillSqlLineApplication.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.client;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import jline.console.completer.StringsCompleter;
+import org.apache.drill.common.scanner.ClassPathScanner;
+import org.apache.drill.common.util.DrillVersionInfo;
+import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
+import sqlline.Application;
+import sqlline.CommandHandler;
+import sqlline.OutputFormat;
+import sqlline.ReflectiveCommandHandler;
+import sqlline.SqlLine;
+import sqlline.SqlLineOpts;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * <p>Drill SqlLine application configuration.</p>
+ *
+ * <p>Customizes SqlLine for Drill, i.e. overrides application info message,
+ * known drivers, connection url examples, removes non applicable commands, sets SqlLine properties.</p>
+ *
+ * <p>Uses {@link #DRILL_SQLLINE_CONF} as base configuration, allows to override it using {@link #DRILL_SQLLINE_OVERRIDE_CONF}.
+ * If configuration files are missing in the classpath, issues warning and proceeds with default SqlLine configuration.</p>
+ */
+public class DrillSqlLineApplication extends Application {
+
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSqlLineApplication.class);
+
+ private static final String DRILL_SQLLINE_CONF = "drill-sqlline.conf";
+ private static final String DRILL_SQLLINE_OVERRIDE_CONF = "drill-sqlline-override.conf";
+
+ private static final String INFO_MESSAGE_TEMPLATE_CONF = "drill.sqlline.info_message_template";
+ private static final String QUOTES_CONF = "drill.sqlline.quotes";
+ private static final String DRIVERS_CONF = "drill.sqlline.drivers";
+ private static final String CONNECTION_URL_EXAMPLES_CONF = "drill.sqlline.connection_url_examples";
+ private static final String COMMANDS_TO_EXCLUDE_CONF = "drill.sqlline.commands.exclude";
+ private static final String OPTS_CONF = "drill.sqlline.opts";
+
+ private final Config config;
+
+ public DrillSqlLineApplication() {
+ this(DRILL_SQLLINE_CONF, DRILL_SQLLINE_OVERRIDE_CONF);
+ }
+
+ @VisibleForTesting
+ public DrillSqlLineApplication(String configName, String overrideConfigName) {
+ this.config = overrideConfig(overrideConfigName, loadConfig(configName));
+ if (config.isEmpty()) {
+ logger.warn("Was unable to find / load [{}]. Will use default SqlLine configuration.", configName);
+ }
+ }
+
+ public Config getConfig() {
+ return config;
+ }
+
+ @Override
+ public String getInfoMessage() throws Exception {
+ if (config.hasPath(INFO_MESSAGE_TEMPLATE_CONF)) {
+ String quote = "";
+ if (config.hasPath(QUOTES_CONF)) {
+ List<String> quotes = config.getStringList(QUOTES_CONF);
+ quote = quotes.get(new Random().nextInt(quotes.size()));
+ }
+ return String.format(config.getString(INFO_MESSAGE_TEMPLATE_CONF), DrillVersionInfo.getVersion(), quote);
+ }
+
+ return super.getInfoMessage();
+ }
+
+ @Override
+ public Collection<String> initDrivers() {
+ if (config.hasPath(DRIVERS_CONF)) {
+ return config.getStringList(DRIVERS_CONF);
+ }
+ return super.initDrivers();
+ }
+
+ @Override
+ public Map<String, OutputFormat> getOutputFormats(SqlLine sqlLine) {
+ return sqlLine.getOutputFormats();
+ }
+
+ @Override
+ public Collection<String> getConnectionUrlExamples() {
+ if (config.hasPath(CONNECTION_URL_EXAMPLES_CONF)) {
+ return config.getStringList(CONNECTION_URL_EXAMPLES_CONF);
+ }
+ return super.getConnectionUrlExamples();
+ }
+
+ @Override
+ public Collection<CommandHandler> getCommandHandlers(SqlLine sqlLine) {
+ List<String> commandsToExclude = new ArrayList<>();
+
+ // exclude connect command and then add it back to ensure connection url examples are updated
+ boolean reloadConnect = config.hasPath(CONNECTION_URL_EXAMPLES_CONF);
+ if (reloadConnect) {
+ commandsToExclude.add("connect");
+ }
+
+ if (config.hasPath(COMMANDS_TO_EXCLUDE_CONF)) {
+ commandsToExclude.addAll(config.getStringList(COMMANDS_TO_EXCLUDE_CONF));
+ }
+
+ if (commandsToExclude.isEmpty()) {
+ return sqlLine.getCommandHandlers();
+ }
+
+ List<CommandHandler> commandHandlers = sqlLine.getCommandHandlers().stream()
+ .filter(c -> c.getNames().stream()
+ .noneMatch(commandsToExclude::contains))
+ .collect(Collectors.toList());
+
+ if (reloadConnect) {
+ commandHandlers.add(new ReflectiveCommandHandler(sqlLine,
+ new StringsCompleter(getConnectionUrlExamples()), "connect", "open"));
+ }
+
+ return commandHandlers;
+ }
+
+ @Override
+ public SqlLineOpts getOpts(SqlLine sqlLine) {
+ SqlLineOpts opts = sqlLine.getOpts();
+ if (config.hasPath(OPTS_CONF)) {
+ Config optsConfig = config.getConfig(OPTS_CONF);
+ optsConfig.entrySet().forEach(
+ e -> {
+ String key = e.getKey();
+ String value = String.valueOf(e.getValue().unwrapped());
+ if (!opts.set(key, value, true)) {
+ logger.warn("Unable to set SqlLine property [{}] to [{}].", key, value);
+ }
+ }
+ );
+ }
+ return opts;
+ }
+
+ private Config loadConfig(String configName) {
+ Set<URL> urls = ClassPathScanner.forResource(configName, false);
+ if (urls.size() != 1) {
+ if (logger.isDebugEnabled()) {
+ urls.forEach(
+ u -> logger.debug("Found duplicating [{}]: [{}].", configName, u.getPath())
+ );
+ }
+ return ConfigFactory.empty();
+ }
+
+ URL url = urls.iterator().next();
+ try {
+ logger.debug("Parsing [{}] for the url: [{}].", configName, url.getPath());
+ return ConfigFactory.parseURL(url);
+ } catch (Exception e) {
+ logger.warn("Was unable to parse [{}].", url.getPath(), e);
+ return ConfigFactory.empty();
+ }
+ }
+
+ private Config overrideConfig(String configName, Config config) {
+ Config overrideConfig = loadConfig(configName);
+ return overrideConfig.withFallback(config).resolve();
+ }
+
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
index 9fe7bea..dd1c5f1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
@@ -55,6 +55,7 @@ import org.apache.zookeeper.Environment;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint.State;
import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
import org.apache.drill.shaded.guava.com.google.common.base.Stopwatch;
+import org.slf4j.bridge.SLF4JBridgeHandler;
/**
* Starts, tracks and stops all the required services for a Drillbit daemon to work.
@@ -70,6 +71,9 @@ public class Drillbit implements AutoCloseable {
*/
GuavaPatcher.patch();
Environment.logEnv("Drillbit environment: ", logger);
+ // Jersey uses java.util.logging - create bridge: jul to slf4j
+ SLF4JBridgeHandler.removeHandlersForRootLogger();
+ SLF4JBridgeHandler.install();
}
public final static String SYSTEM_OPTIONS_NAME = "org.apache.drill.exec.server.Drillbit.system_options";
diff --git a/exec/java-exec/src/main/resources/drill-sqlline.conf b/exec/java-exec/src/main/resources/drill-sqlline.conf
new file mode 100644
index 0000000..b977d9f
--- /dev/null
+++ b/exec/java-exec/src/main/resources/drill-sqlline.conf
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file customizes SqlLine application.
+# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+
+drill.sqlline {
+ info_message_template: "apache drill %s\n%s",
+ quotes: [
+ "start your sql engine",
+ "this isn't your grandfather's sql",
+ "a little sql for your nosql",
+ "json ain't no thang",
+ "drill baby drill",
+ "just drill it",
+ "say hello to my little drill",
+ "what ever the mind of man can conceive and believe, drill can query",
+ "the only truly happy people are children, the creative minority and drill users",
+ "a drill is a terrible thing to waste",
+ "got drill?",
+ "a drill in the hand is better than two in the bush",
+ "drill never goes out of style",
+ "everything is easier with drill",
+ "you told me to drill, sergeant",
+ "you're gonna need a bigger drill",
+ "keep your data close, but your drillbits closer",
+ "let's drill something more solid than concrete",
+ "drill must go on",
+ "in drill we trust",
+ "two things are infinite: the universe and drill; and i'm not sure about the universe",
+ "a query result is never late, nor is it early, it arrives precisely when it means to",
+ "think different, think drill",
+ "there are two types of analysts in the world: those who use drill and those who don't",
+ "good friends, good books and drill cluster: this is the ideal life",
+ "data is the new oil: ready to drill some?",
+ "you won't meet santa, but drill supports clauses",
+ "your drill is the drill that will pierce the heavens"
+ ],
+ drivers: [
+ org.apache.drill.jdbc.Driver
+ ],
+ connection_url_examples: [
+ "jdbc:drill:drillbit=localhost",
+ "jdbc:drill:zk=local"
+ ],
+ commands.exclude: [
+ describe,
+ indexes,
+ primarykeys,
+ exportedkeys,
+ manual,
+ importedkeys,
+ procedures,
+ typeinfo,
+ dropall,
+ metadata,
+ nativesql,
+ dbinfo,
+ rehash,
+ batch,
+ all,
+ go,
+ isolation,
+ autocommit,
+ commit,
+ rollback,
+ scan,
+ sql,
+ call
+ ],
+ opts: {
+ incremental: false,
+ isolation: TRANSACTION_NONE,
+ maxWidth: 10000
+ }
+}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DrillSqlLineApplicationTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DrillSqlLineApplicationTest.java
new file mode 100644
index 0000000..4996ce2
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DrillSqlLineApplicationTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.client;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import sqlline.CommandHandler;
+import sqlline.OutputFormat;
+import sqlline.SqlLine;
+import sqlline.SqlLineOpts;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class DrillSqlLineApplicationTest {
+
+ private static DrillSqlLineApplication application;
+
+ @BeforeClass
+ public static void init() {
+ application = new DrillSqlLineApplication(
+ "drill-sqlline-test.conf",
+ "drill-sqlline-test-override.conf");
+ }
+
+ @Test
+ public void testInfoMessage() throws Exception {
+ String infoMessage = application.getInfoMessage();
+ assertThat(infoMessage, containsString("all code is guilty until proven innocent"));
+ }
+
+ @Test
+ public void testDrivers() {
+ Collection<String> drivers = application.initDrivers();
+ assertEquals(1L, drivers.size());
+ assertEquals("org.apache.drill.jdbc.Driver", drivers.iterator().next());
+ }
+
+ @Test
+ public void testOutputFormats() {
+ SqlLine sqlLine = new SqlLine();
+ Map<String, OutputFormat> outputFormats = application.getOutputFormats(sqlLine);
+ assertEquals(sqlLine.getOutputFormats(), outputFormats);
+ }
+
+ @Test
+ public void testConnectionUrlExamples() {
+ Collection<String> examples = application.getConnectionUrlExamples();
+ assertEquals(1L, examples.size());
+ assertEquals("jdbc:drill:zk=local", examples.iterator().next());
+ }
+
+ @Test
+ public void testCommandHandlers() {
+ SqlLine sqlLine = new SqlLine();
+ Collection<CommandHandler> commandHandlers = application.getCommandHandlers(sqlLine);
+ List<String> excludedCommands = Arrays.asList("describe", "indexes");
+ List<CommandHandler> matchingCommands = commandHandlers.stream()
+ .filter(c -> c.getNames().stream()
+ .anyMatch(excludedCommands::contains))
+ .collect(Collectors.toList());
+ assertTrue(matchingCommands.isEmpty());
+ }
+
+ @Test
+ public void testOpts() {
+ SqlLine sqlLine = new SqlLine();
+ SqlLineOpts opts = application.getOpts(sqlLine);
+ assertFalse(opts.getIncremental());
+ assertEquals("TRANSACTION_NONE", opts.getIsolation());
+ assertEquals(20_000, opts.getMaxWidth());
+ assertEquals(200, opts.getTimeout());
+ }
+
+ @Test
+ public void testEmptyConfig() {
+ DrillSqlLineApplication application = new DrillSqlLineApplication(
+ "missing.conf", "missing_example.conf");
+ assertTrue(application.getConfig().isEmpty());
+ }
+
+ @Test
+ public void testConfigWithoutOverride() {
+ DrillSqlLineApplication application = new DrillSqlLineApplication(
+ "drill-sqlline-test.conf", "missing_example.conf");
+ assertFalse(application.getConfig().isEmpty());
+ }
+
+}
diff --git a/exec/java-exec/src/test/resources/drill-sqlline-test-override.conf b/exec/java-exec/src/test/resources/drill-sqlline-test-override.conf
new file mode 100644
index 0000000..380785c
--- /dev/null
+++ b/exec/java-exec/src/test/resources/drill-sqlline-test-override.conf
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file customizes SqlLine application.
+# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+
+drill.sqlline {
+ quotes: [
+ "all code is guilty until proven innocent"
+ ],
+ connection_url_examples: [
+ "jdbc:drill:zk=local"
+ ],
+ opts: {
+ maxWidth: 20000,
+ timeout: 200
+ }
+}
diff --git a/exec/java-exec/src/test/resources/drill-sqlline-test.conf b/exec/java-exec/src/test/resources/drill-sqlline-test.conf
new file mode 100644
index 0000000..d0d9a89
--- /dev/null
+++ b/exec/java-exec/src/test/resources/drill-sqlline-test.conf
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file customizes SqlLine application.
+# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+
+drill.sqlline {
+ info_message_template: "apache drill %s\n%s",
+ quotes: [
+ "start your sql engine"
+ ],
+ drivers: [
+ org.apache.drill.jdbc.Driver
+ ],
+ connection_url_examples: [
+ "jdbc:drill:drillbit=localhost",
+ "jdbc:drill:zk=local"
+ ],
+ commands.exclude: [
+ describe,
+ indexes
+ ],
+ opts: {
+ incremental: false,
+ isolation: TRANSACTION_NONE,
+ maxWidth: 10000
+ }
+}
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index e5124f0..7a6c9ee 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -168,6 +168,10 @@
<artifactId>metadata-extractor</artifactId>
<groupId>com.drewnoakes</groupId>
</exclusion>
+ <exclusion>
+ <groupId>sqlline</groupId>
+ <artifactId>sqlline</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/pom.xml b/pom.xml
index a6e1f41..4d81927 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,7 +49,7 @@
<calcite.version>1.17.0-drill-r1</calcite.version>
<avatica.version>1.12.0</avatica.version>
<janino.version>2.7.6</janino.version>
- <sqlline.version>1.1.9-drill-r7</sqlline.version>
+ <sqlline.version>1.5.0</sqlline.version>
<jackson.version>2.9.5</jackson.version>
<jackson.databind.version>2.9.5</jackson.databind.version>
<zookeeper.version>3.4.12</zookeeper.version>
@@ -2747,11 +2747,6 @@
<version>2.7.1</version>
</dependency>
<dependency>
- <groupId>sqlline</groupId>
- <artifactId>sqlline</artifactId>
- <version>${sqlline.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>${parquet.version}</version>
[drill] 01/02: DRILL-6625: Intermittent failures in Kafka unit
tests Unit test changes to fix intermittent kafka producer and consumer
errors.
Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sorabh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 6ff854b79f4139237d20bd9a4fbd256f8ce4c0f2
Author: Abhishek Ravi <ab...@gmail.com>
AuthorDate: Mon Sep 10 23:07:12 2018 -0700
DRILL-6625: Intermittent failures in Kafka unit tests
Unit test changes to fix intermittent kafka producer and consumer errors.
- Increase the value of REQUEST_TIMEOUT_MS_CONFIG to accomadate slower systems.
- Increase the value of producer RETRIES_CONFIG to 3 (from 0).
- Prevent producer to send duplicate messages due to retries by enabling Idempotent producer.
- Increase consumer poll timeout (from 200 ms).
- The design of `TestKafkaSuit` is very similar to design of `MongoTestSuit` and hence would require changes similar to the ones made in [storage-mongo/pom.xml](https://github.com/apache/drill/pull/923/commits/f5dfa56f33a46b92e2f9de153d82a16a77642ddf#diff-e110e2cbfd77d27e85d5121529c612bfR83).
- Current behavior is surefire runs test classes twice - once as a part of `TestKafkaSuit` and the other by directly running classes. To prevent the latter from happening, changes were made in `pom.xml` for `storage-mongo` plugin.
closes #1463
---
contrib/storage-kafka/pom.xml | 26 ++++++++++++++++++++++
.../exec/store/kafka/KafkaFilterPushdownTest.java | 5 +++++
.../exec/store/kafka/KafkaMessageGenerator.java | 5 +++--
.../drill/exec/store/kafka/KafkaTestBase.java | 2 +-
.../drill/exec/store/kafka/TestKafkaSuit.java | 1 +
5 files changed, 36 insertions(+), 3 deletions(-)
diff --git a/contrib/storage-kafka/pom.xml b/contrib/storage-kafka/pom.xml
index 5b5917d..0260c1c 100644
--- a/contrib/storage-kafka/pom.xml
+++ b/contrib/storage-kafka/pom.xml
@@ -32,6 +32,7 @@
<properties>
<kafka.version>0.11.0.1</kafka.version>
+ <kafka.TestSuite>**/TestKafkaSuit.class</kafka.TestSuite>
</properties>
<dependencies>
@@ -97,4 +98,29 @@
<scope>test</scope>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <includes>
+ <include>${kafka.TestSuite}</include>
+ </includes>
+ <excludes>
+ <exclude>**/KafkaFilterPushdownTest.java</exclude>
+ <exclude>**/KafkaQueriesTest.java</exclude>
+ <exclude>**/MessageIteratorTest.java</exclude>
+ </excludes>
+ <systemProperties>
+ <property>
+ <name>logback.log.dir</name>
+ <value>${project.build.directory}/surefire-reports</value>
+ </property>
+ </systemProperties>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaFilterPushdownTest.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaFilterPushdownTest.java
index 7be0ec3..d874733 100644
--- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaFilterPushdownTest.java
+++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaFilterPushdownTest.java
@@ -27,6 +27,7 @@ import org.junit.experimental.categories.Category;
import static org.apache.drill.exec.store.kafka.TestKafkaSuit.NUM_JSON_MSG;
import static org.apache.drill.exec.store.kafka.TestKafkaSuit.embeddedKafkaCluster;
+import static org.junit.Assert.assertTrue;
@Category({KafkaStorageTest.class, SlowTest.class})
public class KafkaFilterPushdownTest extends KafkaTestBase {
@@ -42,6 +43,10 @@ public class KafkaFilterPushdownTest extends KafkaTestBase {
KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(),
StringSerializer.class);
generator.populateJsonMsgWithTimestamps(TestQueryConstants.JSON_PUSHDOWN_TOPIC, NUM_JSON_MSG);
+ String query = String.format(TestQueryConstants.MSG_SELECT_QUERY, TestQueryConstants.JSON_PUSHDOWN_TOPIC);
+ //Ensure messages are present
+ assertTrue("Kafka server does not have expected number of messages",
+ testSql(query) == NUM_PARTITIONS * NUM_JSON_MSG);
}
/**
diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaMessageGenerator.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaMessageGenerator.java
index f4a254e..d094531 100644
--- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaMessageGenerator.java
+++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaMessageGenerator.java
@@ -55,14 +55,15 @@ public class KafkaMessageGenerator {
public KafkaMessageGenerator (final String broker, Class<?> valueSerializer) {
producerProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, broker);
producerProperties.put(ProducerConfig.ACKS_CONFIG, "all");
- producerProperties.put(ProducerConfig.RETRIES_CONFIG, 0);
+ producerProperties.put(ProducerConfig.RETRIES_CONFIG, 3);
producerProperties.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
producerProperties.put(ProducerConfig.LINGER_MS_CONFIG, 0);
producerProperties.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 1);
- producerProperties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 1000);
+ producerProperties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000);
producerProperties.put(ProducerConfig.CLIENT_ID_CONFIG, "drill-test-kafka-client");
producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer);
+ producerProperties.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); //So that retries do not cause duplicates
}
public void populateAvroMsgIntoKafka(String topic, int numMsg) throws IOException {
diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java
index 9f06606..b1742d7 100644
--- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java
+++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java
@@ -57,7 +57,7 @@ public class KafkaTestBase extends PlanTestBase {
pluginRegistry.createOrUpdate(KafkaStoragePluginConfig.NAME, storagePluginConfig, true);
testNoResult(String.format("alter session set `%s` = '%s'", ExecConstants.KAFKA_RECORD_READER,
"org.apache.drill.exec.store.kafka.decoders.JsonMessageReader"));
- testNoResult(String.format("alter session set `%s` = %d", ExecConstants.KAFKA_POLL_TIMEOUT, 200));
+ testNoResult(String.format("alter session set `%s` = %d", ExecConstants.KAFKA_POLL_TIMEOUT, 5000));
}
public List<QueryDataBatch> runKafkaSQLWithResults(String sql) throws Exception {
diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java
index ecf998e..784eb4e 100644
--- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java
+++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/TestKafkaSuit.java
@@ -112,6 +112,7 @@ public class TestKafkaSuit {
Properties topicProps = new Properties();
topicProps.put(TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG, "CreateTime");
+ topicProps.put(TopicConfig.RETENTION_MS_CONFIG, "-1");
ZkUtils zkUtils = new ZkUtils(zkClient,
new ZkConnection(embeddedKafkaCluster.getZkServer().getConnectionString()), false);
AdminUtils.createTopic(zkUtils, topicName, partitions, 1,