You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2018/10/15 19:25:47 UTC
[drill] branch master updated (734c460 -> ec6acc7)
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 734c460 DRILL-3988: Expose Drill built-in functions & UDFs in a system table (#1483)
new 0a3cfde DRILL-6473: Update MapR Hive
new 484bcfb DRILL-6777: Setup CircleCI configs for Drill
new 2aede3a DRILL-6783: CAST string literal as INTERVAL MONTH/YEAR works inconsistently when selecting from a table with multiple rows
new ec6acc7 DRILL-6775: The schema for empty output is not shown in Drill Web UI
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.circleci/config.yml | 51 ++++++++++++++++++++++
.../store/hive/readers/HiveAbstractReader.java | 19 +++-----
contrib/storage-hive/hive-exec-shade/pom.xml | 3 ++
contrib/storage-jdbc/pom.xml | 2 +-
.../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 1 +
.../drill/exec/server/rest/QueryWrapper.java | 5 ---
.../drill/exec/server/rest/WebUserConnection.java | 8 ++--
.../drill/exec/fn/impl/TestCastFunctions.java | 38 +++++++++++++++-
.../main/codegen/templates/FixedValueVectors.java | 3 +-
pom.xml | 4 +-
10 files changed, 106 insertions(+), 28 deletions(-)
create mode 100644 .circleci/config.yml
[drill] 01/04: DRILL-6473: Update MapR Hive
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 0a3cfdebd25ea4186c2a2b75524c34323984ffca
Author: Bohdan Kazydub <bo...@gmail.com>
AuthorDate: Mon May 21 19:11:23 2018 +0300
DRILL-6473: Update MapR Hive
close apache/drill#1307
---
.../exec/store/hive/readers/HiveAbstractReader.java | 19 +++++--------------
contrib/storage-hive/hive-exec-shade/pom.xml | 3 +++
pom.xml | 4 ++--
3 files changed, 10 insertions(+), 16 deletions(-)
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
index ba1cd30..5ed6c3b 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
import org.apache.drill.shaded.guava.com.google.common.util.concurrent.ListenableFuture;
import io.netty.buffer.DrillBuf;
@@ -203,20 +204,10 @@ public abstract class HiveAbstractReader extends AbstractRecordReader {
}
}
}
- ColumnProjectionUtils.appendReadColumns(job, columnIds);
-
- // TODO: Use below overloaded method instead of above simpler version of it, once Hive client dependencies
- // (from all profiles) will be updated to 2.3 version or above
-// ColumnProjectionUtils.appendReadColumns(job, columnIds, selectedColumnNames,
-// Lists.newArrayList(Iterables.transform(getColumns(), new Function<SchemaPath, String>()
-// {
-// @Nullable
-// @Override
-// public String apply(@Nullable SchemaPath path)
-// {
-// return path.getRootSegmentPath();
-// }
-// })));
+ List<String> paths = getColumns().stream()
+ .map(SchemaPath::getRootSegmentPath)
+ .collect(Collectors.toList());
+ ColumnProjectionUtils.appendReadColumns(job, columnIds, selectedColumnNames, paths);
for (String columnName : selectedColumnNames) {
StructField fieldRef = finalOI.getStructFieldRef(columnName);
diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml b/contrib/storage-hive/hive-exec-shade/pom.xml
index 3ef1839..b857758 100644
--- a/contrib/storage-hive/hive-exec-shade/pom.xml
+++ b/contrib/storage-hive/hive-exec-shade/pom.xml
@@ -139,6 +139,9 @@
<filter>
<artifact>org.apache.hive:hive-exec</artifact>
<excludes>
+ <!-- This exclusion can be removed once hive-exec uses parquet-hadoop-bundle 1.8.2 or higher.
+ It can be so, for example, after upgrading Hive to 3.0. To check if it's safe to remove the exclusion
+ you can use TestHiveStorage.readFromAlteredPartitionedTableWithEmptyGroupType() test case. -->
<exclude>org/apache/parquet/schema/*</exclude>
</excludes>
</filter>
diff --git a/pom.xml b/pom.xml
index 5d00ee6..3a2d21e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2439,9 +2439,9 @@
<properties>
<alt-hadoop>mapr</alt-hadoop>
<rat.excludeSubprojects>true</rat.excludeSubprojects>
- <hive.version>2.1.1-mapr-1710</hive.version>
+ <hive.version>2.3.3-mapr-1808</hive.version>
<hbase.version>1.1.1-mapr-1602-m7-5.2.0</hbase.version>
- <hadoop.version>2.7.0-mapr-1707</hadoop.version>
+ <hadoop.version>2.7.0-mapr-1808</hadoop.version>
<zookeeper.version>3.4.11-mapr-1808</zookeeper.version>
</properties>
<dependencyManagement>
[drill] 02/04: DRILL-6777: Setup CircleCI configs for Drill
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 484bcfb1e237b6f8f9a244d344f36dcb86dedb61
Author: Vitalii Diravka <vi...@gmail.com>
AuthorDate: Tue Sep 11 14:39:13 2018 +0300
DRILL-6777: Setup CircleCI configs for Drill
- adding .circleci/config.yml to the project to launch CircleCI
- custom memory parameters
- usage of CircleCI machine
- excluding "SlowTest" and "UnlikelyTest" groups
- update maven version
- adding libaio.so library to solve MySQL integration tests
- update com.jcabi:jcabi-mysql-maven-plugin library version
- TODO descriptions for the future enhancements of CircleCI build for Drill
close apache/drill#1493
---
.circleci/config.yml | 51 ++++++++++++++++++++++
contrib/storage-jdbc/pom.xml | 2 +-
.../exec/store/jdbc/TestJdbcPluginWithMySQLIT.java | 1 +
3 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 0000000..1b179df
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+version: 2
+general:
+jobs:
+ build:
+ machine:
+ enabled: true
+ image: circleci/classic:latest
+ parallelism: 1 # TODO: 1. Configuring Parallel Jobs for Running Tests https://circleci.com/docs/2.0/parallelism-faster-jobs/
+
+
+ working_directory: ~/drill
+
+ steps:
+ - checkout
+
+ - run:
+ name: Update maven version
+ # TODO: Could be removed, once Machine Executor image is updated https://github.com/circleci/image-builder/issues/140
+ # and the possibility of specifying Maven version is added https://github.com/circleci/image-builder/issues/143
+ command:
+ curl -fsSL https://git.io/vpDIf | bash -s -- 3.5.4
+ - run:
+ name: Update packages list
+ command:
+ sudo apt-get update
+ - run:
+ name: Install libaio1.so library for MySQL integration tests
+ command:
+ sudo apt-get install libaio1 libaio-dev
+ - run:
+ name: Drill project build
+ # TODO: 2. Optimizing Maven Builds on CircleCI - https://circleci.com/blog/optimizing-maven-builds-on-circleci/
+ # TODO: 3. Caching Dependencies - https://circleci.com/docs/2.0/caching/
+ # TODO: 4. Resolving memory issues without "SlowTest" and "UnlikelyTest" excludedGroups in the build
+ command:
+ mvn install -Drat.skip=false -Dlicense.skip=false -DmemoryMb=2560 -DdirectMemoryMb=4608 -DexcludedGroups="org.apache.drill.categories.SlowTest,org.apache.drill.categories.UnlikelyTest"
diff --git a/contrib/storage-jdbc/pom.xml b/contrib/storage-jdbc/pom.xml
index a032e69..7de89f0 100755
--- a/contrib/storage-jdbc/pom.xml
+++ b/contrib/storage-jdbc/pom.xml
@@ -213,7 +213,7 @@
<plugin>
<groupId>com.jcabi</groupId>
<artifactId>jcabi-mysql-maven-plugin</artifactId>
- <version>0.8.2</version>
+ <version>0.9</version>
<executions>
<execution>
<id>mysql-test</id>
diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
index 1640913..0f377e3 100644
--- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
+++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
@@ -26,6 +26,7 @@ import org.junit.experimental.categories.Category;
/**
* JDBC storage plugin tests against MySQL.
+ * Note: it requires libaio.so library in the system
*/
@Category(JdbcStorageTest.class)
public class TestJdbcPluginWithMySQLIT extends PlanTestBase {
[drill] 03/04: DRILL-6783: CAST string literal as INTERVAL
MONTH/YEAR works inconsistently when selecting from a table with multiple
rows
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 2aede3a1cd1532fbcd0a622b49a3b9fce69979ef
Author: Bohdan Kazydub <bo...@gmail.com>
AuthorDate: Mon Oct 8 16:50:22 2018 +0300
DRILL-6783: CAST string literal as INTERVAL MONTH/YEAR works inconsistently when selecting from a table with multiple rows
close apache/drill#1496
---
.../drill/exec/fn/impl/TestCastFunctions.java | 38 +++++++++++++++++++++-
.../main/codegen/templates/FixedValueVectors.java | 3 +-
2 files changed, 39 insertions(+), 2 deletions(-)
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
index cb0d022..7768909 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
@@ -19,14 +19,21 @@ package org.apache.drill.exec.fn.impl;
import java.math.BigDecimal;
import java.time.LocalDate;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserRemoteException;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.apache.drill.exec.vector.IntervalYearVector;
import org.apache.drill.test.BaseTestQuery;
+import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -39,6 +46,7 @@ import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
import mockit.integration.junit4.JMockit;
import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.hasItem;
@RunWith(JMockit.class)
@Category({UnlikelyTest.class, SqlFunctionTest.class})
@@ -664,4 +672,32 @@ public class TestCastFunctions extends BaseTestQuery {
test(query);
}
-}
\ No newline at end of file
+
+ @Test // DRILL-6783
+ public void testCastVarCharIntervalYear() throws Exception {
+ String query = "select cast('P31M' as interval month) as i from cp.`employee.json` limit 10";
+ List<QueryDataBatch> result = testSqlWithResults(query);
+ RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator());
+
+ QueryDataBatch b = result.get(0);
+ loader.load(b.getHeader().getDef(), b.getData());
+
+ IntervalYearVector vector = (IntervalYearVector) loader.getValueAccessorById(
+ IntervalYearVector.class,
+ loader.getValueVectorId(SchemaPath.getCompoundPath("i")).getFieldIds())
+ .getValueVector();
+
+ Set<String> resultSet = new HashSet<>();
+ for (int i = 0; i < loader.getRecordCount(); i++) {
+ String displayValue = vector.getAccessor().getAsStringBuilder(i).toString();
+ resultSet.add(displayValue);
+ }
+
+ Assert.assertEquals(
+ "Casting literal string as INTERVAL should yield the same result for each row", 1, resultSet.size());
+ Assert.assertThat(resultSet, hasItem("2 years 7 months"));
+
+ b.release();
+ loader.clear();
+ }
+}
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index a98aa66..508e484 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -532,7 +532,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
public StringBuilder getAsStringBuilder(int index) {
- return DateUtilities.intervalYearStringBuilder(data.getInt(index));
+ int value = get(index);
+ return DateUtilities.intervalYearStringBuilder(value);
}
<#elseif minor.class == "Time">
[drill] 04/04: DRILL-6775: The schema for empty output is not shown
in Drill Web UI
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ec6acc7ab6ee1a6eb326a4783189978bf55c0bcf
Author: Anton Gozhiy <an...@gmail.com>
AuthorDate: Tue Oct 9 15:59:16 2018 +0300
DRILL-6775: The schema for empty output is not shown in Drill Web UI
Removed an excess check for the result emptiness that prevented retrieval of the column names.
close apache/drill#1498
---
.../main/java/org/apache/drill/exec/server/rest/QueryWrapper.java | 5 -----
.../java/org/apache/drill/exec/server/rest/WebUserConnection.java | 8 ++++----
2 files changed, 4 insertions(+), 9 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
index 42427aa..4e09858 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
@@ -19,7 +19,6 @@ package org.apache.drill.exec.server.rest;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.exceptions.UserRemoteException;
@@ -115,10 +114,6 @@ public class QueryWrapper {
throw new UserRemoteException(webUserConnection.getError());
}
- if (webUserConnection.results.isEmpty()) {
- webUserConnection.results.add(Maps.<String, String>newHashMap());
- }
-
// Return the QueryResult.
return new QueryResult(queryId, webUserConnection.columns, webUserConnection.results);
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
index df4c8a5..9d6e7e4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
@@ -75,20 +75,20 @@ public class WebUserConnection extends AbstractDisposableUserClientConnection im
@Override
public void sendData(RpcOutcomeListener<Ack> listener, QueryWritableBatch result) {
- // Check if there is any data or not. There can be overflow here but DrillBuf doesn't support allocating with
+ // There can be overflow here but DrillBuf doesn't support allocating with
// bytes in long. Hence we are just preserving the earlier behavior and logging debug log for the case.
final int dataByteCount = (int) result.getByteCount();
- if (dataByteCount <= 0) {
+ if (dataByteCount < 0) {
if (logger.isDebugEnabled()) {
- logger.debug("Either no data received in this batch or there is BufferOverflow in dataByteCount: {}",
+ logger.debug("There is BufferOverflow in dataByteCount: {}",
dataByteCount);
}
listener.success(Acks.OK, null);
return;
}
- // If here that means there is some data for sure. Create a ByteBuf with all the data in it.
+ // Create a ByteBuf with all the data in it.
final int rows = result.getHeader().getRowCount();
final BufferAllocator allocator = webSessionResources.getAllocator();
final DrillBuf bufferWithData = allocator.buffer(dataByteCount);